diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ad0a6d57..595f891c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,7 +303,7 @@ jobs: init-type: ${{ matrix.init-type }} smoke-test-run-ios: - name: Run iOS ${{ matrix.unity-version }} Smoke Test + name: Run iOS ${{ matrix.unity-version }} Integration Test if: ${{ !startsWith(github.ref, 'refs/heads/release/') }} needs: [smoke-test-compile-ios, create-unity-matrix] uses: ./.github/workflows/smoke-test-run-ios.yml @@ -311,6 +311,7 @@ jobs: unity-version: ${{ matrix.unity-version }} ios-version: ${{ matrix.ios-version }} init-type: ${{ matrix.init-type }} + secrets: inherit strategy: fail-fast: false matrix: diff --git a/.github/workflows/smoke-test-build-ios.yml b/.github/workflows/smoke-test-build-ios.yml index dc8563034..4c6209b5d 100644 --- a/.github/workflows/smoke-test-build-ios.yml +++ b/.github/workflows/smoke-test-build-ios.yml @@ -96,9 +96,10 @@ jobs: run: ./test/Scripts.Integration.Test/add-sentry.ps1 -UnityPath "$env:UNITY_PATH" -PackagePath "test-package-release" - name: Configure Sentry - run: ./test/Scripts.Integration.Test/configure-sentry.ps1 -UnityPath "$env:UNITY_PATH" -Platform "$env:BUILD_PLATFORM" -CheckSymbols + run: ./test/Scripts.Integration.Test/configure-sentry.ps1 -UnityPath "$env:UNITY_PATH" -Platform "$env:BUILD_PLATFORM" -CheckSymbols -TestMode "integration" env: BUILD_PLATFORM: ${{ matrix.build_platform }} + SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} - name: Build Project with runtime initialization run: ./test/Scripts.Integration.Test/build-project.ps1 -UnityPath "$env:UNITY_PATH" -Platform "$env:BUILD_PLATFORM" -CheckSymbols:$false -UnityVersion "$env:UNITY_VERSION" @@ -122,9 +123,9 @@ jobs: path: test-app-runtime.tar.gz retention-days: 14 - - name: Overwrite OptionsConfiguration for build-time initialization + - name: Overwrite IntegrationOptionsConfiguration for build-time initialization run: | - $optionsPath = "samples/IntegrationTest/Assets/Scripts/OptionsConfiguration.cs" + $optionsPath = "samples/IntegrationTest/Assets/Scripts/IntegrationOptionsConfiguration.cs" $content = Get-Content $optionsPath -Raw $content = $content -replace 'IosNativeInitializationType = NativeInitializationType.Runtime', 'IosNativeInitializationType = NativeInitializationType.BuildTime' Set-Content $optionsPath $content diff --git a/.github/workflows/smoke-test-run-ios.yml b/.github/workflows/smoke-test-run-ios.yml index 9465e4ded..7d53b345c 100644 --- a/.github/workflows/smoke-test-run-ios.yml +++ b/.github/workflows/smoke-test-run-ios.yml @@ -1,4 +1,4 @@ -name: "SmokeTest: Run iOS" +name: "IntegrationTest: Run iOS" on: workflow_call: inputs: @@ -14,7 +14,7 @@ on: # Map the workflow outputs to job outputs outputs: status: - description: "Smoke test status" + description: "Integration test status" value: ${{ jobs.run.outputs.status }} defaults: @@ -27,16 +27,23 @@ jobs: runs-on: macos-14 # Pinning to get the oldest, supported version of iOS simulator # Map the job outputs to step outputs outputs: - status: ${{ steps.smoke-test.outputs.status }} + status: ${{ steps.integration-test.outputs.status }} env: UNITY_VERSION: ${{ inputs.unity-version }} IOS_VERSION: ${{ inputs.ios-version }} INIT_TYPE: ${{ inputs.init-type }} + ARTIFACTS_PATH: samples/IntegrationTest/test-artifacts/ + SENTRY_TEST_DSN: ${{ secrets.SENTRY_TEST_DSN }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} steps: - name: Checkout uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 + - name: Initialize app-runner submodule + run: git submodule update --init modules/app-runner + shell: bash + - name: Download app artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: @@ -53,13 +60,21 @@ jobs: with: xcode-version: '15.0' # to run iOS 17.0 we need Xcode 15.0 - - name: Run iOS Smoke Tests - id: smoke-test + - name: Run iOS Integration Tests + id: integration-test timeout-minutes: 20 run: | - $runtime = "${env:IOS_VERSION}" - If ($runtime -ne "latest") - { - $runtime = "iOS " + $runtime - } - ./Scripts/smoke-test-ios.ps1 Test "$runtime" -IsIntegrationTest \ No newline at end of file + $env:SENTRY_TEST_APP = "samples/IntegrationTest/Build/IntegrationTest.app" + $env:SENTRY_IOS_VERSION = "${{ inputs.ios-version }}" + Invoke-Pester -Path test/IntegrationTest/Integration.Tests.iOS.ps1 -CI + echo "status=success" >> $env:GITHUB_OUTPUT + + - name: Upload test results on failure + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: testapp-ios-logs-${{ env.IOS_VERSION }}-${{ env.UNITY_VERSION }}-${{ env.INIT_TYPE }} + path: | + ${{ env.ARTIFACTS_PATH }} + test/IntegrationTest/results/ + retention-days: 14 diff --git a/modules/app-runner b/modules/app-runner index 210bbebe2..63d8c36b3 160000 --- a/modules/app-runner +++ b/modules/app-runner @@ -1 +1 @@ -Subproject commit 210bbebe2563bbec1e41b52d9221a3cea20a65c7 +Subproject commit 63d8c36b34580b6b2bdac3311594d07e937773b8 diff --git a/test/IntegrationTest/CommonTestCases.ps1 b/test/IntegrationTest/CommonTestCases.ps1 index c34656851..4f3aab1ea 100644 --- a/test/IntegrationTest/CommonTestCases.ps1 +++ b/test/IntegrationTest/CommonTestCases.ps1 @@ -83,6 +83,12 @@ $CommonTestCases = @( } @{ Name = "Contains app context"; TestBlock = { param($TestSetup, $TestType, $SentryEvent, $RunResult) + + if ($TestType -eq "crash-capture" -and $TestSetup.Platform -eq "iOS") { + Set-ItResult -Skipped -Because "app context is not synced to sentry-cocoa on iOS" + return + } + $SentryEvent.contexts.app | Should -Not -BeNullOrEmpty } } diff --git a/test/IntegrationTest/Integration.Tests.iOS.ps1 b/test/IntegrationTest/Integration.Tests.iOS.ps1 new file mode 100644 index 000000000..b1d52a451 --- /dev/null +++ b/test/IntegrationTest/Integration.Tests.iOS.ps1 @@ -0,0 +1,222 @@ +#!/usr/bin/env pwsh +# +# Integration tests for Sentry Unity SDK (iOS Simulator) +# +# Environment variables: +# SENTRY_TEST_APP: path to the test .app bundle +# SENTRY_IOS_VERSION: iOS simulator version (e.g. "17.0" or "latest") +# SENTRY_TEST_DSN: test DSN +# SENTRY_AUTH_TOKEN: authentication token for Sentry API + +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +# Import app-runner modules +. $PSScriptRoot/../../modules/app-runner/import-modules.ps1 + +# Import shared test cases and utility functions +. $PSScriptRoot/CommonTestCases.ps1 + + +BeforeAll { + $script:BundleId = "com.DefaultCompany.IntegrationTest" + + # Run integration test action on device + function Invoke-TestAction { + param ( + [Parameter(Mandatory=$true)] + [string]$Action + ) + + Write-Host "Running $Action..." + + $appArgs = @("--test", $Action) + + $runResult = Invoke-DeviceApp -ExecutablePath $script:BundleId -Arguments $appArgs + + # Save result to JSON file + $runResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "${Action}-result.json") + + # Launch app again to ensure crash report is sent + if ($Action -eq "crash-capture") { + Write-Host "Running crash-send to ensure crash report is sent..." + + $sendArgs = @("--test", "crash-send") + $sendResult = Invoke-DeviceApp -ExecutablePath $script:BundleId -Arguments $sendArgs + + # Save crash-send result to JSON for debugging + $sendResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "crash-send-result.json") + + # Print crash-send output + Write-Host "::group::App output (crash-send)" + $sendResult.Output | ForEach-Object { Write-Host $_ } + Write-Host "::endgroup::" + + # Attach to runResult for test access + $runResult | Add-Member -NotePropertyName "CrashSendOutput" -NotePropertyValue $sendResult.Output + } + + # Print app output so it's visible in CI logs + Write-Host "::group::App output ($Action)" + $runResult.Output | ForEach-Object { Write-Host $_ } + Write-Host "::endgroup::" + + return $runResult + } + + # Create directory for the test results + New-Item -ItemType Directory -Path "$PSScriptRoot/results/" -ErrorAction Continue 2>&1 | Out-Null + Set-OutputDir -Path "$PSScriptRoot/results/" + + # Initialize test parameters + $script:TestSetup = [PSCustomObject]@{ + Platform = "iOS" + AppPath = $env:SENTRY_TEST_APP + iOSVersion = $env:SENTRY_IOS_VERSION + Dsn = $env:SENTRY_TEST_DSN + AuthToken = $env:SENTRY_AUTH_TOKEN + } + + # Validate environment + if ([string]::IsNullOrEmpty($script:TestSetup.AppPath)) { + throw "SENTRY_TEST_APP environment variable is not set." + } + if (-not (Test-Path $script:TestSetup.AppPath)) { + throw "App not found at: $($script:TestSetup.AppPath)" + } + if ([string]::IsNullOrEmpty($script:TestSetup.iOSVersion)) { + throw "SENTRY_IOS_VERSION environment variable is not set." + } + if ([string]::IsNullOrEmpty($script:TestSetup.Dsn)) { + throw "SENTRY_TEST_DSN environment variable is not set." + } + if ([string]::IsNullOrEmpty($script:TestSetup.AuthToken)) { + throw "SENTRY_AUTH_TOKEN environment variable is not set." + } + + Connect-SentryApi ` + -ApiToken $script:TestSetup.AuthToken ` + -DSN $script:TestSetup.Dsn + + $target = $script:TestSetup.iOSVersion + # Convert bare version numbers (e.g. "17.0") to "iOS 17.0" format expected by iOSSimulatorProvider + if ($target -match '^\d+\.\d+$') { + $target = "iOS $target" + } + Connect-Device -Platform "iOSSimulator" -Target $target + Install-DeviceApp -Path $script:TestSetup.AppPath +} + + +AfterAll { + Disconnect-SentryApi + Disconnect-Device +} + + +Describe "Unity iOS Integration Tests" { + + Context "Message Capture" { + BeforeAll { + $script:runEvent = $null + $script:runResult = Invoke-TestAction -Action "message-capture" + + $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 + if ($eventId) { + Write-Host "::group::Getting event content" + $script:runEvent = Get-SentryTestEvent -EventId "$eventId" + Write-Host "::endgroup::" + } + } + + It "" -ForEach $CommonTestCases { + & $testBlock -SentryEvent $runEvent -TestType "message-capture" -RunResult $runResult -TestSetup $script:TestSetup + } + + It "Has message level info" { + ($runEvent.tags | Where-Object { $_.key -eq "level" }).value | Should -Be "info" + } + + It "Has message content" { + $runEvent.title | Should -Not -BeNullOrEmpty + } + } + + Context "Exception Capture" { + BeforeAll { + $script:runEvent = $null + $script:runResult = Invoke-TestAction -Action "exception-capture" + + $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 + if ($eventId) { + Write-Host "::group::Getting event content" + $script:runEvent = Get-SentryTestEvent -EventId "$eventId" + Write-Host "::endgroup::" + } + } + + It "" -ForEach $CommonTestCases { + & $testBlock -SentryEvent $runEvent -TestType "exception-capture" -RunResult $runResult -TestSetup $script:TestSetup + } + + It "Has exception information" { + $runEvent.exception | Should -Not -BeNullOrEmpty + $runEvent.exception.values | Should -Not -BeNullOrEmpty + } + + It "Has exception with stacktrace" { + $exception = $runEvent.exception.values[0] + $exception | Should -Not -BeNullOrEmpty + $exception.type | Should -Not -BeNullOrEmpty + $exception.stacktrace | Should -Not -BeNullOrEmpty + } + + It "Has error level" { + ($runEvent.tags | Where-Object { $_.key -eq "level" }).value | Should -Be "error" + } + } + + Context "Crash Capture" { + BeforeAll { + $script:runEvent = $null + $script:runResult = Invoke-TestAction -Action "crash-capture" + + $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 + if ($eventId) { + Write-Host "::group::Getting event content" + $script:runEvent = Get-SentryTestEvent -TagName "test.crash_id" -TagValue "$eventId" -TimeoutSeconds 120 + Write-Host "::endgroup::" + } + } + + It "" -ForEach $CommonTestCases { + & $testBlock -SentryEvent $runEvent -TestType "crash-capture" -RunResult $runResult -TestSetup $script:TestSetup + } + + It "Has fatal level" { + ($runEvent.tags | Where-Object { $_.key -eq "level" }).value | Should -Be "fatal" + } + + It "Has exception with stacktrace" { + $runEvent.exception | Should -Not -BeNullOrEmpty + $runEvent.exception.values | Should -Not -BeNullOrEmpty + $exception = $runEvent.exception.values[0] + $exception | Should -Not -BeNullOrEmpty + $exception.stacktrace | Should -Not -BeNullOrEmpty + } + + It "Reports crashedLastRun as Crashed on relaunch" { + $crashedLastRunLine = $runResult.CrashSendOutput | Where-Object { + $_ -match "crashedLastRun=Crashed" + } + $crashedLastRunLine | Should -Not -BeNullOrEmpty -Because "Native SDK should report crashedLastRun=Crashed after a native crash" + } + + It "Crash-send completes flush successfully" { + $flushLine = $runResult.CrashSendOutput | Where-Object { + $_ -match "Flush complete" + } + $flushLine | Should -Not -BeNullOrEmpty -Because "crash-send should complete its flush before quitting" + } + } +} diff --git a/test/IntegrationTest/Integration.Tests.ps1 b/test/IntegrationTest/Integration.Tests.ps1 index dc36f22c0..3b4d32be4 100644 --- a/test/IntegrationTest/Integration.Tests.ps1 +++ b/test/IntegrationTest/Integration.Tests.ps1 @@ -69,6 +69,7 @@ BeforeAll { # Initialize test parameters $script:TestSetup = [PSCustomObject]@{ + Platform = "Android" ApkPath = $env:SENTRY_TEST_APK Dsn = $env:SENTRY_TEST_DSN AuthToken = $env:SENTRY_AUTH_TOKEN