diff --git a/AzureDevOps/DistributeTests.ps1 b/AzureDevOps/DistributeTests.ps1 new file mode 100644 index 0000000..c5f9672 --- /dev/null +++ b/AzureDevOps/DistributeTests.ps1 @@ -0,0 +1,43 @@ +<# +.SYNOPSIS + Distribute tests across multiple agents in VSTS pipeline +.DESCRIPTION + This script divides test files across multiple agents for faster execution. It searches for files matching a specific pattern (for example, `test*`) and assigns them based on the agent number. + For example, if there are multiple files [test1..test10] and 2 agents: + - Agent 1 runs tests from odd-numbered files. + - Agent 2 runs tests from even-numbered files. + For detailed slicing information, see https://learn.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner?view=azure-devops +#> + +$tests = Get-ChildItem .\tests\ -Filter *.m -File # Search for test files matching the specified pattern +$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # Standard VSTS variable containing the number of parallel jobs +$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE # Current job position +$testCount = $tests.Count + +# Handle cases where the pipeline runs without parallel configuration (single agent) +if ($totalAgents -eq 0) { + $totalAgents = 1 +} +if (!$agentNumber -or $agentNumber -eq 0) { + $agentNumber = 1 +} + +Write-Host "Total agents: $totalAgents" +Write-Host "Agent number: $agentNumber" +Write-Host "Total tests: $testCount" + +$testsToRun= @() +# Slice test files so each agent gets a unique set of files to execute +For ($i=$agentNumber; $i -le $testCount;) { + $file = $tests[$i-1] + + $fileName = [System.IO.Path]::GetFileNameWithoutExtension($file.Name) + $testsToRun += "$fileName/*" + $i = $i + $totalAgents + } + +# Join all test files into a space-separated string +$testFiles = $testsToRun -Join " " +Write-Host "Tests to run $testFiles" +# Write files into a variable for execution in a subsequent task +Write-Host "##vso[task.setvariable variable=MATLABTestFiles;]$testFiles" \ No newline at end of file diff --git a/AzureDevOps/DistributeTests.sh b/AzureDevOps/DistributeTests.sh new file mode 100644 index 0000000..f029379 --- /dev/null +++ b/AzureDevOps/DistributeTests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +#=============================================================================== +# +# FILE: distribute_tests.sh +# +# USAGE: ./distribute_tests.sh +# +# DESCRIPTION: This script divides test files across multiple agents for faster execution. It searches for files matching a specific pattern (for example, `test*`) and assigns them based on the agent number. +# For example, if there are multiple files [test1..test10] and 2 agents: +# - Agent 1 runs tests from odd-numbered files. +# - Agent 2 runs tests from even-numbered files. +# For detailed slicing information, see https://learn.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner?view=azure-devops +# +#=============================================================================== + +tests=() +# Search for test files matching the specified pattern +while IFS= read -r file; do + tests+=("$file") +done < <(find ./tests -type f -name "*.m" | sort) + +totalAgents=${SYSTEM_TOTALJOBSINPHASE} # Standard VSTS variable containing the number of parallel jobs +agentNumber=${SYSTEM_JOBPOSITIONINPHASE} # Current job position +testCount=${#tests[@]} + +# Handle cases where the pipeline runs without parallel configuration (single agent) +if [ $totalAgents -eq 0 ]; then + totalAgents=1 +fi +if [ -z $agentNumber ]; then + agentNumber=1 +fi + +echo "Total agents: $totalAgents" +echo "Agent number: $agentNumber" +echo "Total tests: $testCount" + +testsToRun=() +# Slice test files so each agent gets a unique set of files +for (( i=agentNumber; i<=testCount; i+=totalAgents )); do + file="${tests[i-1]}" + + fileName=$(basename "$file" .m) + testsToRun+=("${fileName}/*") +done + +# Join all test files into a space-separated string +testFiles="${testsToRun[*]}" +echo "Tests to run $testFiles" + +# Write files into a variable for execution in a subsequent task +echo "##vso[task.setvariable variable=MATLABTestFiles;]$testFiles" \ No newline at end of file diff --git a/AzureDevOps/ParallelStrategy.yml b/AzureDevOps/ParallelStrategy.yml new file mode 100644 index 0000000..ce72d0f --- /dev/null +++ b/AzureDevOps/ParallelStrategy.yml @@ -0,0 +1,84 @@ +jobs: + - job: ParallelWindows + # Specify 'parallel' strategy to run tests in parallel + strategy: + parallel: 2 + pool: + vmImage: windows-latest + steps: + # Install MATLAB and required products + - task: InstallMATLAB@1 + inputs: + products: MATLAB_Compiler_SDK MATLAB_Test + + - task: RunMATLABBuild@1 + inputs: + tasks: mex buildPythonPackage + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) + + - powershell: .\AzureDevOps\DistributeTests.ps1 + displayName: 'PowerShell script to distribute tests' + + - task: RunMATLABTests@1 + inputs: + selectByName: $(MATLABTestFiles) + sourceFolder: src + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) + + - job: ParallelLinux + # Specify 'parallel' strategy to run tests in parallel + strategy: + parallel: 2 + pool: + vmImage: ubuntu-latest + steps: + # Install MATLAB and required products + - task: InstallMATLAB@1 + inputs: + products: MATLAB_Compiler_SDK MATLAB_Test + + - task: RunMATLABBuild@1 + inputs: + tasks: mex buildPythonPackage + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) + + - bash: chmod +x ./AzureDevOps/DistributeTests.sh && ./AzureDevOps/DistributeTests.sh + displayName: 'Bash script to distribute tests' + + - task: RunMATLABTests@1 + inputs: + selectByName: $(MATLABTestFiles) + sourceFolder: src + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) + + - job: ParallelMac + # Specify 'parallel' strategy to run tests in parallel + strategy: + parallel: 2 + pool: + vmImage: macOS-latest + steps: + # Install MATLAB and required products + - task: InstallMATLAB@1 + inputs: + products: MATLAB_Compiler_SDK MATLAB_Test + + - task: RunMATLABBuild@1 + inputs: + tasks: mex buildPythonPackage + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) + + - bash: chmod +x ./AzureDevOps/DistributeTests.sh && ./AzureDevOps/DistributeTests.sh + displayName: 'Bash script to distribute tests' + + - task: RunMATLABTests@1 + inputs: + selectByName: $(MATLABTestFiles) + sourceFolder: src + env: + MLM_LICENSE_TOKEN: $(MLM_LICENSE_TOKEN) \ No newline at end of file diff --git a/README.md b/README.md index 18ea2c7..6b0816d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ For starter workflows, use the [`ci-configuration-examples`](https://github.com/ # Workflows -The repository contains examples for packaging and distributing a toolbox, as well as building and uploading Python® packages. +The repository contains examples for packaging and distributing a toolbox, building and uploading Python® packages, and running tests across multiple build agents. - **Package and Distribute Toolbox**: Using a matrix build across multiple platforms, compile, link, and test your C source files to produce a binary MEX file per operating system. Then, bundle the resulting binaries into a toolbox and distribute it as a GitHub release. @@ -36,6 +36,8 @@ The repository contains examples for packaging and distributing a toolbox, as we | GitHub Actions| [`.github/workflows/CrossPlatformBuilder.yml`](https://github.com/mathworks/advanced-ci-configuration-examples/blob/main/.github/workflows/CrossPlatformBuilder.yml) | | Jenkins | [`Jenkins/CrossPlatformBuilder/Jenkinsfile`](https://github.com/mathworks/advanced-ci-configuration-examples/blob/main/Jenkins/CrossPlatformBuilder/Jenkinsfile) | +- **Run Tests Across Multiple Agents**: Use the [parallel strategy](https://learn.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner?view=azure-devops) in Azure DevOps to run tests across multiple agents and speed up the testing process. For configuration details, see the example in [`AzureDevOps/ParallelStrategy.yml`](https://github.com/mathworks/advanced-ci-configuration-examples/blob/main/AzureDevOps/ParallelStrategy.yml). +
## Get Started