Merge pull request #533 from bugsnag/https

run e2e tests over https

Author: djskinner

.buildkite/browser-pipeline.full.yml

@@ -29,6 +29,7 @@ steps:
         run: browser-maze-runner-bs
         use-aliases: true
         command:
+          - --https
           - --farm=bs
           - --browser={{ matrix }}
       artifacts#v1.5.0:

.buildkite/browser-pipeline.yml

@@ -46,6 +46,7 @@ steps:
             use-aliases: true
             service-ports: true
             command:
+              - --https
               - --farm=bb
               - --browser={{ matrix.browser }}_{{ matrix.version }}
               - --appium-version=1.22
@@ -78,6 +79,7 @@ steps:
             run: browser-maze-runner-legacy
             use-aliases: true
             command:
+              - --https
               - --farm=bs
               - --browser={{ matrix.browser }}_{{ matrix.version }}
           artifacts#v1.5.0:
@@ -97,5 +99,31 @@ steps:
           adjustments:
             - with: { browser: firefox, version: 60 }
             - with: { browser: edge, version: 80 }
-            - with: { browser: safari, version: 11 }
             - with: { browser: edge, version: latest }
+
+      # BrowserStack non-https
+      - label: ":browserstack: :{{ matrix.browser }}: {{ matrix.version }} Browser non-https tests${EXTRA_STEP_LABEL}"
+        depends_on: "browser-maze-runner-legacy-${USE_CDN_BUILD}"
+        timeout_in_minutes: 30
+        plugins:
+          docker-compose#v4.12.0:
+            pull: browser-maze-runner-legacy
+            run: browser-maze-runner-legacy
+            use-aliases: true
+            command:
+              - --farm=bs
+              - --browser={{ matrix.browser }}_{{ matrix.version }}
+          artifacts#v1.5.0:
+            upload:
+              - "./test/browser/maze_output/failed/**/*"
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "BROWSER_PERFORMANCE_BUILDKITE_ANALYTICS_TOKEN"
+        concurrency: 2
+        concurrency_group: "browserstack"
+        matrix:
+          setup:
+            browser: [safari]
+            version: [11]

test/browser/Gemfile.lock

@@ -11,13 +11,13 @@ brew install chromedriver
 To run all the tests, run the following in `test/browser`:
 
 ```sh
-bundle exec maze-runner --farm=local --browser=chrome
+bundle exec maze-runner --https --farm=local --browser=chrome
 ```
 
 Or to run a single feature file:
 
 ```sh
-bundle exec maze-runner --farm=local --browser=chrome features/device.feature
+bundle exec maze-runner --https --farm=local --browser=chrome features/device.feature
 ```
 
 ## Running the end-to-end tests with BrowserStack
@@ -34,11 +34,11 @@ The browsers available to test on are the keys in [`browsers.yml`](https://githu
 To run all the tests, run the following in `test/browser`:
 
 ```sh
-bundle exec maze-runner --farm=bs --browser=chrome_latest
+bundle exec maze-runner --https --farm=bs --browser=chrome_latest
 ```
 
 Or to run a single feature file:
 
 ```sh
-bundle exec maze-runner --farm=bs --browser=chrome_latest features/manual-spans.feature
+bundle exec maze-runner --https --farm=bs --browser=chrome_latest features/manual-spans.feature
 ```

test/browser/TESTING.md

@@ -5,7 +5,7 @@
 Feature: Angular
 
     Scenario: Angular route change spans are automatically instrumented
-        Given I navigate to the test URL "/angular/dist"
+        Given I navigate to the test URL "/docs/angular/dist"
         And the element "customers" is present
         And I click the element "customers"
         When I wait to receive 1 trace
@@ -25,7 +25,7 @@ Feature: Angular
             | bugsnag.browser.page.route_change.trigger | stringValue  | imperative             |
 
     Scenario: Route with a custom URL matcher
-        Given I navigate to the test URL "/angular/dist"
+        Given I navigate to the test URL "/docs/angular/dist"
         And the element "customMatching" is present
         And I click the element "customMatching"
         When I wait to receive 1 trace

test/browser/features/angular.feature

@@ -1,7 +1,7 @@
 Feature: Custom attributes
 
     Scenario: Custom attributes can be added to spans
-        Given I navigate to the test URL "/custom-attributes"
+        Given I navigate to the test URL "/docs/custom-attributes"
         And I wait for 1 span
 
         Then a span named "Custom/CustomAttributesScenario" contains the attributes: 

test/browser/features/custom-attributes.feature

@@ -2,13 +2,13 @@ Feature: Device ID persistence
 
   Scenario: persisted device ID is used if present
     # load the page and persist an ID
-    Given I navigate to the test URL "/retry-scenario"
+    Given I navigate to the test URL "/docs/retry-scenario"
     And I wait to receive a sampling request
     And I store the device ID "c1234567890abcdefghijklmnop"
 
     # reload the page to prove the ID was persisted and not just available in
     # memory somehow
-    When I navigate to the test URL "/retry-scenario"
+    When I navigate to the test URL "/docs/retry-scenario"
     And I wait to receive a sampling request
     And I click the element "send-first-span"
     And I wait for 1 span

test/browser/features/device-id.feature

@@ -1,11 +1,11 @@
 Feature: Configuration
 
   Scenario: Delivery is prevented when releaseStage is not in enabledReleaseStages
-    Given I navigate to the test URL "/enabled-release-stages-disabled"
+    Given I navigate to the test URL "/docs/enabled-release-stages-disabled"
     Then I should receive no traces
 
   Scenario: Delivery is allowed when releaseStage is present in enabledReleaseStages
-    Given I navigate to the test URL "/enabled-release-stages"
+    Given I navigate to the test URL "/docs/enabled-release-stages"
     And I wait to receive at least 1 trace
     Then the trace payload field "resourceSpans.0.scopeSpans.0.spans.0.name" equals "Custom/Should send"
     And the trace payload field "resourceSpans.0.resource" string attribute "deployment.environment" equals "test"

test/browser/features/enabled-release-stages.feature

@@ -1,15 +1,15 @@
 Feature: Error Correlation
 
   Scenario: Error does not include the correlation property when no spans are open
-    Given I navigate to the test URL "/error-correlation"
+    Given I navigate to the test URL "/docs/error-correlation"
     And I wait to receive a sampling request
     And I click the element "send-error"
     And I wait to receive an error
     Then the error is a valid browser payload for the error reporting API
     And the error payload field "events.0.correlation" is null
 
   Scenario: Reported errors include the current trace and span id
-    Given I navigate to the test URL "/error-correlation"
+    Given I navigate to the test URL "/docs/error-correlation"
     And I wait to receive a sampling request
     Then I click the element "start-span"
     And I wait for 1 second
@@ -33,7 +33,7 @@ Feature: Error Correlation
     And the error payload field "events.0.correlation.traceId" equals the stored value "traceId"
 
   Scenario: Reported errors do not include the trace and span id if the span is not the current context
-    Given I navigate to the test URL "/error-correlation"
+    Given I navigate to the test URL "/docs/error-correlation"
     And I wait to receive a sampling request
     Then I click the element "start-span"
     And I wait for 1 second

test/browser/features/error-correlation.feature

@@ -1,7 +1,7 @@
 Feature: Fixed Sampling Probability
 
   Scenario: Uses configured sampling probability
-    Given I navigate to the test URL "/fixed-sampling-probability"
+    Given I navigate to the test URL "/docs/fixed-sampling-probability"
     And I enter unmanaged traces mode
     When I click the element "send-span"
     And I wait to receive 1 trace