diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx b/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx index 00f328b35c35..d348dd7df6b3 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/CommandDefinitionStrings.resx @@ -454,6 +454,24 @@ This is equivalent to deleting project.assets.json. List the discovered tests instead of running the tests. Optionally accepts a format: 'text' (default) for human-readable output or 'json' for machine-readable output. + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + Run tests and write the source-to-test map used by affected-test selection. + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + Run only tests linked in the test map to sources changed in Git. + Don't allow updating project lock file. diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs b/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs index 61ff9e11af13..85f0e4cc6f6e 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/Commands/Test/TestCommandDefinition.MicrosoftTestingPlatform.cs @@ -8,6 +8,7 @@ using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.CommandLine; using Microsoft.DotNet.Cli.Help; +using Microsoft.DotNet.Cli.Utils; namespace Microsoft.DotNet.Cli.Commands.Test; @@ -181,6 +182,18 @@ public sealed partial class MicrosoftTestingPlatform : TestCommandDefinition, IC Arity = ArgumentArity.Zero }; + public const string EnableAffectedTestsEnvironmentVariable = "DOTNET_CLI_ENABLE_AFFECTED_TESTS"; + + public const string CollectTestMapOptionName = "--collect-test-map"; + + public readonly Option CollectTestMapOption; + + public const string AffectedTestsOptionName = "--affected-tests"; + + public readonly Option AffectedTestsOption; + + public bool AffectedTestsEnabled { get; } + public readonly Option ArtifactsPathOption = CommonOptions.CreateArtifactsPathOption(); public const string BuildTargetName = "_MTPBuild"; @@ -195,6 +208,24 @@ public MicrosoftTestingPlatform() MinimumExpectedTestsOption.Validators.Add(ValidatePositiveInteger); MaximumFailedTestsOption.Validators.Add(ValidatePositiveInteger); + AffectedTestsEnabled = EnvironmentVariableParser.ParseBool( + Environment.GetEnvironmentVariable(EnableAffectedTestsEnvironmentVariable), + defaultValue: false); + + CollectTestMapOption = new(CollectTestMapOptionName) + { + Description = CommandDefinitionStrings.CmdCollectTestMapDescription, + Arity = ArgumentArity.Zero, + Hidden = !AffectedTestsEnabled, + }; + + AffectedTestsOption = new(AffectedTestsOptionName) + { + Description = CommandDefinitionStrings.CmdAffectedTestsDescription, + Arity = ArgumentArity.Zero, + Hidden = !AffectedTestsEnabled, + }; + Options.Add(ProjectOrSolutionOption); Options.Add(SolutionOption); Options.Add(TestModulesFilterOption); @@ -228,7 +259,33 @@ public MicrosoftTestingPlatform() Options.Add(NoLaunchProfileArgumentsOption); Options.Add(DeviceOption); Options.Add(ListDevicesOption); + Options.Add(CollectTestMapOption); + Options.Add(AffectedTestsOption); Options.Add(MTPTargetOption); + + Validators.Add(commandResult => + { + bool collectTestMap = commandResult.HasOption(CollectTestMapOption); + bool affectedTests = commandResult.HasOption(AffectedTestsOption); + if (!AffectedTestsEnabled && (collectTestMap || affectedTests)) + { + commandResult.AddError(string.Format( + CommandDefinitionStrings.CmdAffectedTestsFeatureDisabled, + EnableAffectedTestsEnvironmentVariable)); + } + else if (collectTestMap && affectedTests) + { + commandResult.AddError(CommandDefinitionStrings.CmdAffectedTestsOptionsMutuallyExclusive); + } + else if (collectTestMap && commandResult.HasOption(MaxParallelTestModulesOption)) + { + commandResult.AddError(CommandDefinitionStrings.CmdCollectTestMapCannotRunModulesInParallel); + } + else if (collectTestMap && commandResult.HasOption(MinimumExpectedTestsOption)) + { + commandResult.AddError(CommandDefinitionStrings.CmdCollectTestMapCannotRequireMinimumTests); + } + }); } public IEnumerable> CustomHelpLayout() diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf index cba8dbe23022..edf1edf27e01 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.cs.xlf @@ -157,6 +157,21 @@ Cílový modul runtime pro vyčištění. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Umožní shromažďovat výpisy stavu systému při očekávaných i neočekávaných ukončeních hostitele testů. @@ -244,6 +259,21 @@ Při použití společně s testy řízenými daty závisí chování časového Pro MSTest před 2.2.4 se časový limit použije pro všechny testovací případy. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf index c3dd440f7dd7..51810544fa59 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.de.xlf @@ -157,6 +157,21 @@ Die Zielruntime für die Bereinigung. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Aktiviert die Erfassung von Absturzabbildern bei einer erwarteten und einer unerwarteten Beendigung des Testhosts. @@ -244,6 +259,21 @@ Wenn dies zusammen mit datengesteuerten Tests verwendet wird, hängt das Timeout Für MSTest vor 2.2.4 wird das Timeout für alle Testfälle verwendet. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf index a08e0a16bbf8..a68b5f56c63f 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.es.xlf @@ -157,6 +157,21 @@ El entorno de tiempo de ejecución para el que se limpia. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Habilita la recopilación del volcado de memoria en la salida del host de prueba esperada e inesperada. @@ -244,6 +259,21 @@ Cuando se usa junto con pruebas basadas en datos, el comportamiento del tiempo d Para MSTest antes de 2.2.4, el tiempo de espera se usa para todos los casos de prueba. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf index 842088ce7893..55ec13ed1da4 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.fr.xlf @@ -157,6 +157,21 @@ Runtime cible pour lequel le nettoyage est effectué. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Active la collecte des données de vidage sur plantage en cas de sortie attendue et inattendue de testhost. @@ -244,6 +259,21 @@ Lorsqu’elle est utilisée avec des tests pilotés par les données, le comport Pour MSTest avant la version 2.2.4, le délai d’expiration est utilisé pour tous les cas de test. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf index 58c6f741bd63..215816a95fe0 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.it.xlf @@ -157,6 +157,21 @@ Runtime di destinazione per cui eseguire la pulizia. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Abilita la raccolta del dump di arresto anomalo in caso di chiusura prevista e imprevista dell'host di test. @@ -244,6 +259,21 @@ Se viene usato insieme a test basati sui dati, il comportamento del timeout dipe Per MSTest anteriore a 2.2.4, il timeout viene usato per tutti i test case. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf index 572b6f6eea39..d12dac127b4d 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ja.xlf @@ -157,6 +157,21 @@ クリーンする対象のターゲット ランタイム。 + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. TestHost の予期されるおよび予期されない終了時にクラッシュ ダンプを収集することを有効にします。 @@ -244,6 +259,21 @@ For MSTest before 2.2.4, the timeout is used for all testcases. 2.2.4 より前の MSTest の場合、タイムアウトはすべてのテストケースで使用されます。 + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf index b77b47bac76c..646858e3e4f1 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ko.xlf @@ -157,6 +157,21 @@ 정리할 대상 런타임입니다. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. 예상된 테스트 호스트 종료와 예기치 않은 테스트 호스트 종료 시 크래시 덤프 수집을 사용하도록 설정합니다. @@ -244,6 +259,21 @@ For MSTest before 2.2.4, the timeout is used for all testcases. MSTest 2.2.4 이전의 경우 시간 제한은 모든 테스트케이스에 사용됩니다. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf index b214042fe2d9..f5a134435a0b 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pl.xlf @@ -157,6 +157,21 @@ Docelowe środowisko uruchomieniowe czyszczenia. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Włącza zbieranie zrzutów awaryjnych po oczekiwanym i nieoczekiwanym zakończenia działania przez host testowy. @@ -244,6 +259,21 @@ W przypadku użycia razem z testami opartymi na danych zachowanie limitu czasu z W przypadku platformy MSTest w wersji wcześniejszej niż 2.2.4 limit czasu jest używany dla wszystkich przypadków testowych. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf index 17a9a6ff0751..129291f22f34 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.pt-BR.xlf @@ -157,6 +157,21 @@ O runtime de destino para o qual a limpeza ocorrerá. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Habilita a coleta de despejo de memória nas saídas esperada e inesperada do host de teste. @@ -244,6 +259,21 @@ Quando usado junto com testes controlados por dados, o comportamento do tempo li Para MSTest antes de 2.2.4, o tempo limite é usado para todos os casos de teste. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf index 334f13f1d11c..d06cc9029657 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.ru.xlf @@ -157,6 +157,21 @@ Целевая среда выполнения для очистки. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Включает сбор аварийного дампа при ожидаемом и неожиданном завершении работы узла тестирования. @@ -244,6 +259,21 @@ For MSTest before 2.2.4, the timeout is used for all testcases. В MSTest версии ниже 2.2.4 время ожидания подсчитывается суммарно для всех тестовых случаев. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf index 18a805219b9f..47343f55cff8 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.tr.xlf @@ -157,6 +157,21 @@ Temizlenecek hedef çalışma zamanı. + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. Hem beklenen hem de beklenmeyen test ana bilgisayarı çıkışında kilitlenme bilgi dökümünün toplanmasını sağlar. @@ -244,6 +259,21 @@ Veri odaklı testlerle birlikte kullanıldığında, zaman aşımı davranışı MSTest için 2.2.4'ten önce, zaman aşımı tüm test durumları için kullanılır. + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf index be82176f72d1..ec456cbe079b 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hans.xlf @@ -157,6 +157,21 @@ 要清理的目标运行时。 + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. 允许在预期和意外的 testhost 退出时收集故障转储。 @@ -244,6 +259,21 @@ For MSTest before 2.2.4, the timeout is used for all testcases. 对于 2.2.4 之前的 MSTest，超时用于所有测试用例。 + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf index 081bec89a742..84db1c0d1844 100644 --- a/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf +++ b/src/Cli/Microsoft.DotNet.Cli.Definitions/xlf/CommandDefinitionStrings.zh-Hant.xlf @@ -157,6 +157,21 @@ 要為其進行清理的目標執行階段。 + + Run only tests linked in the test map to sources changed in Git. + Run only tests linked in the test map to sources changed in Git. + + + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + Enables collecting crash dump on expected as well as unexpected testhost exit. 允許在測試主機如預期或未預期地結束時收集損毀傾印。 @@ -244,6 +259,21 @@ For MSTest before 2.2.4, the timeout is used for all testcases. 針對 2.2.4 之前的 MSTest，系統會針對所有測試案例使用逾時。 + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + + Run tests and write the source-to-test map used by affected-test selection. + Run tests and write the source-to-test map used by affected-test selection. + + CONFIG_FILE CONFIG_FILE diff --git a/src/Cli/dotnet/Commands/CliCommandStrings.resx b/src/Cli/dotnet/Commands/CliCommandStrings.resx index 528ed6781554..7d75261ffcf7 100644 --- a/src/Cli/dotnet/Commands/CliCommandStrings.resx +++ b/src/Cli/dotnet/Commands/CliCommandStrings.resx @@ -456,6 +456,9 @@ This is equivalent to deleting project.assets.json. The '--list-devices' and '--list-tests' options cannot be used together. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. @@ -2753,4 +2756,19 @@ Proceed? (Specified in '{0}') {0} is the path to the global.json file. + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + diff --git a/src/Cli/dotnet/Commands/Test/MTP/ITestHandler.cs b/src/Cli/dotnet/Commands/Test/MTP/ITestHandler.cs index 1837797db18b..4a54bb11000a 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/ITestHandler.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/ITestHandler.cs @@ -7,5 +7,7 @@ internal interface ITestHandler { bool Initialize(); + IEnumerable GetTestApplicationWorkingDirectories(); + int RunTestApplications(TestApplicationActionQueue actionQueue); } diff --git a/src/Cli/dotnet/Commands/Test/MTP/MSBuildHandler.cs b/src/Cli/dotnet/Commands/Test/MTP/MSBuildHandler.cs index 97538c9ce283..3455f12f7c90 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/MSBuildHandler.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/MSBuildHandler.cs @@ -80,6 +80,10 @@ public int RunTestApplications(TestApplicationActionQueue actionQueue) return actionQueue.CompleteEnqueueAndWait(); } + public IEnumerable GetTestApplicationWorkingDirectories() + => _testApplications.SelectMany(static group => group) + .Select(static module => module.RunProperties.WorkingDirectory); + private static void LogProjectProperties(IEnumerable moduleGroups) { if (!Logger.TraceEnabled) diff --git a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs index d8b6f5dc73cc..130aebd03d25 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Immutable; using System.CommandLine; using System.Runtime.CompilerServices; using Microsoft.Build.Definition; @@ -17,11 +18,29 @@ namespace Microsoft.DotNet.Cli.Commands.Test; internal partial class MicrosoftTestingPlatformTestCommand { + private const string MinimumExpectedTestsOptionName = "--minimum-expected-tests"; + public int Run(ParseResult parseResult, bool isHelp) { var definition = (TestCommandDefinition.MicrosoftTestingPlatform)parseResult.CommandResult.Command; + string invocationWorkingDirectory = Directory.GetCurrentDirectory(); BuildOptions buildOptions = MSBuildUtility.GetBuildOptions(parseResult); + (buildOptions, bool forwardedCollectTestMap, bool forwardedAffectedTests) = + NormalizeForwardedAffectedTestsOptions(buildOptions); + bool forwardedMinimumExpectedTests = HasForwardedOption( + buildOptions.TestApplicationArguments, + MinimumExpectedTestsOptionName); + + bool collectTestMap = parseResult.HasOption(definition.CollectTestMapOption) || forwardedCollectTestMap; + bool affectedTests = parseResult.HasOption(definition.AffectedTestsOption) || forwardedAffectedTests; + ValidateAffectedTestsOptions( + definition, + parseResult, + collectTestMap, + affectedTests, + forwardedMinimumExpectedTests); + ValidationUtility.ValidateMutuallyExclusiveOptions(parseResult, buildOptions.PathOptions); // --list-devices and --list-tests describe incompatible behaviors: the former lists @@ -31,6 +50,11 @@ public int Run(ParseResult parseResult, bool isHelp) throw new GracefulException(CliCommandStrings.CmdListDevicesAndListTestsMutuallyExclusive); } + if (buildOptions.ListDevices && (collectTestMap || affectedTests)) + { + throw new GracefulException(CliCommandStrings.CmdListDevicesAndAffectedTestsMutuallyExclusive); + } + // --list-devices and --device require a project to evaluate; --test-modules bypasses // project evaluation entirely, so the combination is meaningless. if (buildOptions.PathOptions.TestModules is not null @@ -80,6 +104,17 @@ public int Run(ParseResult parseResult, bool isHelp) return ExitCode.GenericFailure; } + (bool responseFileCollectTestMap, bool responseFileAffectedTests, bool responseFileMinimumExpectedTests) = + DetectAffectedTestsOptionsInForwardedResponseFiles( + buildOptions.TestApplicationArguments, + testHandler.GetTestApplicationWorkingDirectories(), + invocationWorkingDirectory); + collectTestMap |= responseFileCollectTestMap; + affectedTests |= responseFileAffectedTests; + forwardedCollectTestMap |= responseFileCollectTestMap; + forwardedAffectedTests |= responseFileAffectedTests; + forwardedMinimumExpectedTests |= responseFileMinimumExpectedTests; + // Ends the session on the success path, so a failure MSBuild only reports from EndBuild - // a binary logger failing to write, for example - is surfaced rather than swallowed. buildSession.Complete(); @@ -92,12 +127,25 @@ public int Run(ParseResult parseResult, bool isHelp) logger?.ReallyShutdown(); } - int degreeOfParallelism = GetDegreeOfParallelism(parseResult); + ValidateAffectedTestsOptions( + definition, + parseResult, + collectTestMap, + affectedTests, + forwardedMinimumExpectedTests); + + int degreeOfParallelism = GetDegreeOfParallelism(parseResult, collectTestMap); var testOptions = new TestOptions( IsHelp: isHelp, IsDiscovery: parseResult.HasOption(definition.ListTestsOption), - ListTestsFormat: GetListTestsFormat(parseResult, definition)); + ListTestsFormat: GetListTestsFormat(parseResult, definition)) + { + CollectTestMap = collectTestMap, + AffectedTests = affectedTests, + CollectTestMapForwarded = forwardedCollectTestMap, + AffectedTestsForwarded = forwardedAffectedTests, + }; var output = InitializeOutput(degreeOfParallelism, parseResult, testOptions); using var testRunPolicy = new TestRunPolicy( @@ -161,7 +209,7 @@ public int Run(ParseResult parseResult, bool isHelp) else if (exitCode == ExitCode.Success && !isHelp && !parseResult.HasOption(definition.MinimumExpectedTestsOption) && - output.TotalTests == 0) + ShouldFailForNoExecutedTests(testOptions.IsAffectedTestsMode, output.TotalTests, output.SkippedTests)) { // Whole-run "zero tests ran" verdict. Individual modules that matched no tests return exit // code 8, but TestApplicationActionQueue normalizes that to success so a single empty module @@ -202,6 +250,316 @@ internal static bool ShouldPostProcessArtifacts( && !cancellationRequested && cancellationReason == TestRunCancellationReason.None; + internal static (BuildOptions BuildOptions, bool CollectTestMap, bool AffectedTests) NormalizeForwardedAffectedTestsOptions( + BuildOptions buildOptions) + { + bool collectTestMap = false; + bool affectedTests = false; + ImmutableArray.Builder remainingArguments = ImmutableArray.CreateBuilder(); + foreach (string argument in buildOptions.TestApplicationArguments) + { + if (IsAffectedTestsOption(argument, TestCommandDefinition.MicrosoftTestingPlatform.CollectTestMapOptionName)) + { + collectTestMap = true; + remainingArguments.Add(argument); + } + else if (IsAffectedTestsOption(argument, TestCommandDefinition.MicrosoftTestingPlatform.AffectedTestsOptionName)) + { + affectedTests = true; + remainingArguments.Add(argument); + } + else + { + remainingArguments.Add(argument); + } + } + + return ( + buildOptions with { TestApplicationArguments = remainingArguments.ToImmutable() }, + collectTestMap, + affectedTests); + } + + private static void ValidateAffectedTestsOptions( + TestCommandDefinition.MicrosoftTestingPlatform definition, + ParseResult parseResult, + bool collectTestMap, + bool affectedTests, + bool forwardedMinimumExpectedTests) + { + if (!definition.AffectedTestsEnabled && (collectTestMap || affectedTests)) + { + throw new GracefulException( + string.Format( + CliCommandStrings.CmdAffectedTestsFeatureDisabled, + TestCommandDefinition.MicrosoftTestingPlatform.EnableAffectedTestsEnvironmentVariable)); + } + + if (collectTestMap && affectedTests) + { + throw new GracefulException(CliCommandStrings.CmdAffectedTestsOptionsMutuallyExclusive); + } + + if (collectTestMap && parseResult.HasOption(definition.MaxParallelTestModulesOption)) + { + throw new GracefulException(CliCommandStrings.CmdCollectTestMapCannotRunModulesInParallel); + } + + if (collectTestMap && + (parseResult.HasOption(definition.MinimumExpectedTestsOption) || forwardedMinimumExpectedTests)) + { + throw new GracefulException(CliCommandStrings.CmdCollectTestMapCannotRequireMinimumTests); + } + } + + internal static (bool CollectTestMap, bool AffectedTests, bool MinimumExpectedTests) DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray testApplicationArguments, + IEnumerable testApplicationWorkingDirectories, + string invocationWorkingDirectory) + { + var workingDirectories = testApplicationWorkingDirectories + .Select(directory => string.IsNullOrEmpty(directory) + ? invocationWorkingDirectory + : Path.GetFullPath(directory, invocationWorkingDirectory)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + ForwardedOptionState? commonState = null; + bool foundInvalidResponseFile = false; + foreach (string workingDirectory in workingDirectories) + { + ForwardedOptionState workingDirectoryState = default; + foreach (string argument in testApplicationArguments) + { + if (argument.Length > 1 && argument[0] == '@') + { + if (!TryDetectAffectedTestsOptionsInResponseFile( + argument[1..], + workingDirectory, + new HashSet(StringComparer.Ordinal), + out ForwardedOptionState responseFileState)) + { + foundInvalidResponseFile = true; + continue; + } + + workingDirectoryState = workingDirectoryState.Merge(responseFileState); + } + } + + if (commonState is { } previousState && + (previousState.CollectTestMap != workingDirectoryState.CollectTestMap || + previousState.AffectedTests != workingDirectoryState.AffectedTests)) + { + throw new GracefulException(CliCommandStrings.CmdAffectedTestsResponseFilesMustBeConsistent); + } + + commonState = workingDirectoryState with + { + MinimumExpectedTests = + (commonState?.MinimumExpectedTests ?? false) || workingDirectoryState.MinimumExpectedTests, + }; + } + + ForwardedOptionState state = commonState ?? default; + if (foundInvalidResponseFile && (state.CollectTestMap || state.AffectedTests)) + { + throw new GracefulException(CliCommandStrings.CmdAffectedTestsResponseFilesMustBeConsistent); + } + + if (foundInvalidResponseFile) + { + // MTP will report the response-file error. Do not partially activate a mode + // or replace its diagnostic with an SDK validation error. + return default; + } + + return (state.CollectTestMap, state.AffectedTests, state.MinimumExpectedTests); + } + + private static bool TryDetectAffectedTestsOptionsInResponseFile( + string responseFilePath, + string workingDirectory, + HashSet recursionStack, + out ForwardedOptionState state) + { + state = default; + string fullPath = Path.GetFullPath(responseFilePath, workingDirectory); + if (!recursionStack.Add(fullPath) || !File.Exists(fullPath)) + { + return false; + } + + try + { + string[] tokens = [.. + File.ReadAllLines(fullPath) + .Select(static line => line.Trim()) + .Where(static line => line.Length > 0 && line[0] != '#') + .SelectMany(SplitResponseFileLine)]; + + ForwardedOptionState detectedState = default; + foreach (string token in tokens) + { + if (token.Length > 1 && token[0] == '@') + { + if (!TryDetectAffectedTestsOptionsInResponseFile( + token[1..], + workingDirectory, + recursionStack, + out ForwardedOptionState nestedState)) + { + return false; + } + + detectedState = detectedState.Merge(nestedState); + } + else + { + if (IsAffectedTestsOption(token, TestCommandDefinition.MicrosoftTestingPlatform.CollectTestMapOptionName)) + { + detectedState = detectedState with { CollectTestMap = true }; + } + else if (IsAffectedTestsOption(token, TestCommandDefinition.MicrosoftTestingPlatform.AffectedTestsOptionName)) + { + detectedState = detectedState with { AffectedTests = true }; + } + else if (IsOption(token, MinimumExpectedTestsOptionName, allowValue: true)) + { + detectedState = detectedState with { MinimumExpectedTests = true }; + } + } + } + + state = detectedState; + return true; + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or FormatException) + { + // MTP reports response-file read and format errors. Do not replace its diagnostic here. + return false; + } + finally + { + recursionStack.Remove(fullPath); + } + } + + private static bool HasForwardedOption(ImmutableArray arguments, string canonicalOption) + => arguments.Any(argument => IsOption(argument, canonicalOption, allowValue: true)); + + private static bool IsAffectedTestsOption(string argument, string canonicalOption) + => IsOption(argument, canonicalOption, allowValue: false); + + private static bool IsOption(string argument, string canonicalOption, bool allowValue) + { + if (argument.Length < 2 || + argument[0] != '-' || + (argument[1] == '-' && (argument.Length < 3 || argument[2] == '-'))) + { + return false; + } + + string option = argument[1] == '-' ? argument[2..] : argument[1..]; + int separatorIndex = option.IndexOfAny('=', ':'); + if (separatorIndex >= 0 && !allowValue) + { + return false; + } + + ReadOnlySpan optionName = separatorIndex >= 0 ? option.AsSpan(0, separatorIndex) : option; + return optionName.Equals(canonicalOption.AsSpan().TrimStart('-'), StringComparison.OrdinalIgnoreCase); + } + + private static IEnumerable SplitResponseFileLine(string line) + { + int tokenStart = 0; + int position = 0; + bool seekingTokenStart = true; + bool insideQuotes = false; + + while (position < line.Length) + { + char character = line[position]; + + if (char.IsWhiteSpace(character)) + { + if (!insideQuotes) + { + if (!seekingTokenStart) + { + yield return CurrentToken(); + tokenStart = position; + seekingTokenStart = true; + } + else + { + tokenStart = position; + } + } + } + if (character == '"') + { + if (seekingTokenStart) + { + if (insideQuotes) + { + yield return CurrentToken(); + tokenStart = position; + insideQuotes = false; + } + else + { + tokenStart = position + 1; + insideQuotes = true; + } + } + else + { + insideQuotes = !insideQuotes; + } + } + else if (seekingTokenStart && !insideQuotes && !char.IsWhiteSpace(character)) + { + seekingTokenStart = false; + tokenStart = position; + } + + position++; + + if (position == line.Length) + { + if (insideQuotes) + { + throw new FormatException(); + } + + if (!seekingTokenStart) + { + yield return CurrentToken(); + } + } + } + + string CurrentToken() => line.Substring(tokenStart, position - tokenStart).Replace("\"", string.Empty); + } + + private readonly record struct ForwardedOptionState( + bool CollectTestMap, + bool AffectedTests, + bool MinimumExpectedTests) + { + public ForwardedOptionState Merge(ForwardedOptionState other) + => new( + CollectTestMap || other.CollectTestMap, + AffectedTests || other.AffectedTests, + MinimumExpectedTests || other.MinimumExpectedTests); + } + + internal static bool ShouldFailForNoExecutedTests(bool isAffectedTestsMode, int totalTests, int skippedTests) + => (!isAffectedTestsMode && totalTests == 0) || + (totalTests > 0 && totalTests == skippedTests); + private static TestListFormat GetListTestsFormat(ParseResult parseResult, TestCommandDefinition.MicrosoftTestingPlatform definition) { // '--list-tests' has ZeroOrOne arity. A bare '--list-tests' (no value) defaults to text. @@ -257,6 +615,7 @@ private static TerminalTestReporter InitializeOutput(int degreeOfParallelism, Pa ShowAssembly = !isJsonDiscovery, ShowAssemblyStartAndComplete = !isJsonDiscovery, MinimumExpectedTests = parseResult.GetValue(definition.MinimumExpectedTestsOption), + AllowZeroTests = testOptions.IsAffectedTestsMode, ListTestsFormat = testOptions.ListTestsFormat, }); @@ -271,8 +630,13 @@ private static TerminalTestReporter InitializeOutput(int degreeOfParallelism, Pa return output; } - private static int GetDegreeOfParallelism(ParseResult parseResult) + private static int GetDegreeOfParallelism(ParseResult parseResult, bool collectTestMap) { + if (collectTestMap) + { + return 1; + } + var definition = (TestCommandDefinition.MicrosoftTestingPlatform)parseResult.CommandResult.Command; var degreeOfParallelism = parseResult.GetValue(definition.MaxParallelTestModulesOption); diff --git a/src/Cli/dotnet/Commands/Test/MTP/Options.cs b/src/Cli/dotnet/Commands/Test/MTP/Options.cs index 3c9425ea8639..4cf806181afe 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Options.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Options.cs @@ -18,7 +18,22 @@ internal enum TestListFormat Json, } -internal record TestOptions(bool IsHelp, bool IsDiscovery, TestListFormat ListTestsFormat, bool IsArtifactPostProcessing = false); +internal record TestOptions( + bool IsHelp, + bool IsDiscovery, + TestListFormat ListTestsFormat, + bool IsArtifactPostProcessing = false) +{ + internal const string AffectedTestsModeEnvironmentVariable = "DOTNET_CLI_TEST_AFFECTED_TESTS_MODE"; + internal const string CollectTestMapMode = "collect"; + internal const string RunAffectedTestsMode = "run"; + + public bool CollectTestMap { get; init; } + public bool AffectedTests { get; init; } + public bool CollectTestMapForwarded { get; init; } + public bool AffectedTestsForwarded { get; init; } + public bool IsAffectedTestsMode => CollectTestMap || AffectedTests; +} internal record PathOptions(string? ProjectOrSolutionPath, string? SolutionPath, string? TestModules, string? ResultsDirectoryPath, string? ConfigFilePath, string? DiagnosticOutputDirectoryPath); diff --git a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs index b11716472178..c3703d57cedc 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs @@ -72,6 +72,7 @@ internal sealed partial class TerminalTestReporter : IDisposable public bool HasHandshakeFailure => _handshakeFailuresCount > 0; public int TotalTests => _assemblies.Values.Sum(a => a.TotalTests); + public int SkippedTests => _assemblies.Values.Sum(a => a.SkippedTests); // Specifying no timeout, the regex is linear. And the timeout does not measure the regex only, but measures also any // thread suspends, so the regex gets blamed incorrectly. @@ -267,10 +268,15 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode) int totalSkippedTests = _assemblies.Values.Sum(a => a.SkippedTests); bool notEnoughTests = totalTests < _options.MinimumExpectedTests; - bool allTestsWereSkipped = totalTests == 0 || totalTests == totalSkippedTests; + bool allTestsWereSkipped = (totalTests == 0 && !_options.AllowZeroTests) + || (totalTests > 0 && totalTests == totalSkippedTests); bool anyTestFailed = totalFailedTests > 0; bool anyAssemblyFailed = _assemblies.Values.Any(a => !a.Success) || HasHandshakeFailure; - bool runFailed = anyAssemblyFailed || anyTestFailed || notEnoughTests || allTestsWereSkipped || _wasCancelled; + bool unexpectedNonZeroExitCode = exitCode is not null + && exitCode != ExitCode.Success + && exitCode != ExitCode.ZeroTests + && exitCode != ExitCode.MinimumExpectedTestsPolicyViolation; + bool runFailed = anyAssemblyFailed || anyTestFailed || notEnoughTests || allTestsWereSkipped || unexpectedNonZeroExitCode || _wasCancelled; terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen); terminal.Append(CliCommandStrings.TestRunSummary); @@ -284,7 +290,7 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode) { terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests)); } - else if (anyTestFailed || HasHandshakeFailure) + else if (anyTestFailed || HasHandshakeFailure || unexpectedNonZeroExitCode) { // Handshake failures take precedence over "Zero tests ran": when an assembly failed to // hand-shake we want the headline to reflect that the run failed, not that no tests ran @@ -464,12 +470,12 @@ private static void AppendExitCodeAndUrl(ITerminal terminal, int? exitCode, bool /// /// Print a build result summary to the output. /// - private static void AppendAssemblyResult(ITerminal terminal, TestProgressState state) + private void AppendAssemblyResult(ITerminal terminal, TestProgressState state) { if (state.ExitCode == ExitCode.ZeroTests) { - terminal.SetColor(TerminalColor.DarkRed); - terminal.Append(CliCommandStrings.ZeroTestsRan); + terminal.SetColor(_options.AllowZeroTests ? TerminalColor.DarkGreen : TerminalColor.DarkRed); + terminal.Append(_options.AllowZeroTests ? CliCommandStrings.PassedLowercase : CliCommandStrings.ZeroTestsRan); terminal.ResetColor(); } else if (!state.Success) @@ -858,7 +864,7 @@ internal void AssemblyRunCompleted(string executionId, _terminalWithProgress.WriteToTerminal(terminal => AppendAssemblySummary(assemblyRun, terminal)); } - if (exitCode == 0) + if (exitCode == 0 || (_options.AllowZeroTests && exitCode == ExitCode.ZeroTests)) { // Report nothing, we don't want to report on success, because then we will also report on test-discovery etc. return; @@ -969,7 +975,7 @@ void AppendOutputWhenPresent(string description, string? output) // escape char .Replace('\x001b', '\x241b'); - private static void AppendAssemblySummary(TestProgressState assemblyRun, ITerminal terminal) + private void AppendAssemblySummary(TestProgressState assemblyRun, ITerminal terminal) { terminal.ResetColor(); diff --git a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporterOptions.cs b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporterOptions.cs index 33584ba6fc9c..1961cf8b692e 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporterOptions.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporterOptions.cs @@ -30,6 +30,11 @@ internal sealed class TerminalTestReporterOptions /// public int MinimumExpectedTests { get; init; } + /// + /// Gets a value indicating whether a run with no selected tests is successful. + /// + public bool AllowZeroTests { get; init; } + /// /// Gets a value indicating whether we should write the progress periodically to screen. When ANSI is allowed we update the progress as often as we can. /// When ANSI is not allowed we never have progress. diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs index 5afbe20dae26..f513de2fccc3 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs @@ -265,7 +265,7 @@ public async Task RunAsync(CtrlCCancellationManager ctrlC) } } - private ProcessStartInfo CreateProcessStartInfo() + internal ProcessStartInfo CreateProcessStartInfo() { var processStartInfo = new ProcessStartInfo { @@ -312,6 +312,19 @@ private ProcessStartInfo CreateProcessStartInfo() processStartInfo.Environment[Module.DotnetRootArchVariableName] = Path.GetDirectoryName(new Muxer().MuxerPath); } + if (TestOptions.CollectTestMap) + { + processStartInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] = TestOptions.CollectTestMapMode; + } + else if (TestOptions.AffectedTests) + { + processStartInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] = TestOptions.RunAffectedTestsMode; + } + else + { + processStartInfo.Environment.Remove(TestOptions.AffectedTestsModeEnvironmentVariable); + } + processStartInfo.Environment["DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY"] = Directory.GetCurrentDirectory(); return processStartInfo; } @@ -349,6 +362,16 @@ _artifactPostProcessingInvocation is null builder.Append($" {TestCommandDefinition.MicrosoftTestingPlatform.ListTestsOptionName}"); } + if (TestOptions.CollectTestMap && !TestOptions.CollectTestMapForwarded) + { + builder.Append($" {TestCommandDefinition.MicrosoftTestingPlatform.CollectTestMapOptionName}"); + } + + if (TestOptions.AffectedTests && !TestOptions.AffectedTestsForwarded) + { + builder.Append($" {TestCommandDefinition.MicrosoftTestingPlatform.AffectedTestsOptionName}"); + } + if (_buildOptions.PathOptions.ResultsDirectoryPath is { } resultsDirectoryPath) { builder.Append($" {TestCommandDefinition.MicrosoftTestingPlatform.ResultsDirectoryOptionName} {ArgumentEscaper.EscapeSingleArg(resultsDirectoryPath)}"); diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestModulesFilterHandler.cs b/src/Cli/dotnet/Commands/Test/MTP/TestModulesFilterHandler.cs index b2b5944dca57..01863bd24b01 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestModulesFilterHandler.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestModulesFilterHandler.cs @@ -86,6 +86,11 @@ public int RunTestApplications(TestApplicationActionQueue actionQueue) return actionQueue.CompleteEnqueueAndWait(); } + public IEnumerable GetTestApplicationWorkingDirectories() + { + yield return null; + } + internal static List GetMatchedModulePaths(string testModules, string? rootDirectory) { if (string.IsNullOrEmpty(rootDirectory)) diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.cs.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.cs.xlf index eb4e83d016ee..0f933e995a09 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.cs.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.cs.xlf @@ -107,11 +107,41 @@ .NET Builder + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.de.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.de.xlf index 0e1c6b40701c..6ff6600b6558 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.de.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.de.xlf @@ -107,11 +107,41 @@ .NET-Generator + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.es.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.es.xlf index 37e796e6228e..7d324012b52b 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.es.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.es.xlf @@ -107,11 +107,41 @@ Generador para .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.fr.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.fr.xlf index 5f6d4e40370b..170d62e2f1c9 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.fr.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.fr.xlf @@ -107,11 +107,41 @@ Générateur .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.it.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.it.xlf index 12f807d2c9c2..7f1c7aed1a70 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.it.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.it.xlf @@ -107,11 +107,41 @@ Generatore .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ja.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ja.xlf index 6f48eb35367b..117a66173a88 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ja.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ja.xlf @@ -107,11 +107,41 @@ .NET ビルダー + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ko.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ko.xlf index f9f6fc9dea15..f02f7e51c1d5 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ko.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ko.xlf @@ -107,11 +107,41 @@ .NET 작성기 + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pl.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pl.xlf index 2edcb897b92b..23158a79af50 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pl.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pl.xlf @@ -107,11 +107,41 @@ Konstruktor platformy .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pt-BR.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pt-BR.xlf index eb85b23fa16c..1dbfd112306f 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pt-BR.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.pt-BR.xlf @@ -107,11 +107,41 @@ Construtor do .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ru.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ru.xlf index 1ec82dc8c312..58d1b292638f 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ru.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.ru.xlf @@ -107,11 +107,41 @@ Построитель .NET + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.tr.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.tr.xlf index b8f27352e45a..7dd8806ac186 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.tr.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.tr.xlf @@ -107,11 +107,41 @@ .NET Oluşturucusu + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hans.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hans.xlf index 2c1fbddb789a..488dadb9d047 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hans.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hans.xlf @@ -107,11 +107,41 @@ .NET 生成器 + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hant.xlf b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hant.xlf index ed5ff733529b..f97df28c7cb2 100644 --- a/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hant.xlf +++ b/src/Cli/dotnet/Commands/xlf/CliCommandStrings.zh-Hant.xlf @@ -107,11 +107,41 @@ .NET 產生器 + + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + Affected-test selection is experimental. Set the '{0}' environment variable to '1' to enable it. + + + + The options '--collect-test-map' and '--affected-tests' cannot be used together. + The options '--collect-test-map' and '--affected-tests' cannot be used together. + + + + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + Forwarded response files must select the same affected-test operation for every test application. Specify '--collect-test-map' or '--affected-tests' directly on 'dotnet test' instead. + + + + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + The option '--collect-test-map' cannot be combined with '--minimum-expected-tests'. Collection batches do not report test totals to the parent process. + + + + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + The option '--collect-test-map' cannot be combined with '--max-parallel-test-modules'. Test maps are collected one module at a time. + + The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. The '--device' and '--list-devices' options require a project and cannot be used with '--test-modules'. + + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + The '--list-devices' option cannot be combined with '--collect-test-map' or '--affected-tests'. + + The '--list-devices' and '--list-tests' options cannot be used together. The '--list-devices' and '--list-tests' options cannot be used together. diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index b686ffdc246d..7858febcdce2 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -698,9 +698,7 @@ at Microsoft.DotNet.Cli.Commands.Test.TestApplicationActionQueue.Read(BuildOptio result.StdErr.Should().Contain("System.InvalidOperationException: A test session start event was received without a corresponding test session end."); - // TODO: It's much better to introduce a new kind of "summary" indicating - // that the test app exited with zero exit code before sending test session end event - result.StdOut.Should().Contain("Test run summary: Passed!") + result.StdOut.Should().Contain("Test run summary: Failed!") .And.Contain("total: 1") .And.Contain("succeeded: 1") .And.Contain("failed: 0") diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestSelectsDevice.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestSelectsDevice.cs index 94c18e367ca7..ec965c799f6a 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestSelectsDevice.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestSelectsDevice.cs @@ -441,6 +441,24 @@ public void ItErrorsWhenListDevicesAndListTestsAreCombined() .And.HaveStdErrContaining(CliCommandStrings.CmdListDevicesAndListTestsMutuallyExclusive); } + [TestMethod] + [DataRow("--collect-test-map")] + [DataRow("--affected-tests")] + public void ItErrorsWhenListDevicesAndAffectedTestOperationAreCombined(string affectedTestOption) + { + var testInstance = TestAssetsManager.CopyTestAsset("DotnetTestDevices", $"ListDevicesWith{affectedTestOption.TrimStart('-')}") + .WithSource(); + + var result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE", "en-US") + .WithEnvironmentVariable("DOTNET_CLI_ENABLE_AFFECTED_TESTS", "1") + .Execute("--list-devices", affectedTestOption, "-f", "net11.0-android"); + + result.Should().Fail() + .And.HaveStdErrContaining(CliCommandStrings.CmdListDevicesAndAffectedTestsMutuallyExclusive); + } + [TestMethod] public void ItListsDevicesForExplicitFrameworkOnMultiTargetedProject() { diff --git a/test/dotnet.Tests/CommandTests/Test/MTPHelpSnapshotTests.cs b/test/dotnet.Tests/CommandTests/Test/MTPHelpSnapshotTests.cs index 5b4211a49ee3..0aaf916269cf 100644 --- a/test/dotnet.Tests/CommandTests/Test/MTPHelpSnapshotTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/MTPHelpSnapshotTests.cs @@ -26,6 +26,7 @@ public async Task VerifyMTPHelpOutput() CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable(TestCommandDefinition.MicrosoftTestingPlatform.EnableAffectedTestsEnvironmentVariable, "0") .Execute(CliConstants.HelpOptionKey); result.ExitCode.Should().Be(ExitCodes.Success); diff --git a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs index b96fcb0e6acb..0afbbbdd45e3 100644 --- a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs @@ -199,6 +199,87 @@ public void TestExecutionCompleted_WithZeroTestsAndPassingAssemblies_PrintsPasse output.Should().NotContain("error:"); } + [TestMethod] + public void TestExecutionCompleted_WithAllowedZeroTests_PrintsPassingAssemblyAndRunSummary() + { + var capturingConsole = new CapturingConsole(); + var options = new TerminalTestReporterOptions + { + AllowZeroTests = true, + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + ShowAssembly = true, + ShowAssemblyStartAndComplete = true, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + + const string assembly = "/repo/bin/Debug/net9.0/Affected.Tests.dll"; + reporter.AssemblyRunStarted(assembly, "net9.0", "x64", executionId: "exec-empty", instanceId: "inst-empty"); + reporter.AssemblyRunCompleted( + executionId: "exec-empty", + exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.ZeroTests, + outputData: null, + errorData: null); + reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.Success); + + string output = StripAnsi(capturingConsole.GetOutput()); + output.Should().Contain("Test run summary: Passed!"); + GetAssemblySummaryLine(output, assembly).Should().Contain("passed"); + output.Should().NotContain("Zero tests ran"); + output.Should().NotContain("Test run returned non-zero exit code"); + } + + [TestMethod] + public void TestExecutionCompleted_WithAllowedZeroTestsAndAllSelectedTestsSkipped_RemainsZeroTests() + { + var capturingConsole = new CapturingConsole(); + var options = new TerminalTestReporterOptions + { + AllowZeroTests = true, + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + ShowAssembly = true, + ShowAssemblyStartAndComplete = false, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + + const string assembly = "/repo/bin/Debug/net9.0/Affected.Tests.dll"; + reporter.AssemblyRunStarted(assembly, "net9.0", "x64", executionId: "exec-skipped", instanceId: "inst-skipped"); + ReportTest(reporter, assembly, executionId: "exec-skipped", instanceId: "inst-skipped", testUid: "skipped-1", TestOutcome.Skipped); + reporter.AssemblyRunCompleted( + executionId: "exec-skipped", + exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.Success, + outputData: null, + errorData: null); + reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.Success); + + StripAnsi(capturingConsole.GetOutput()).Should().Contain("Zero tests ran"); + } + + [TestMethod] + public void TestExecutionCompleted_WithAllowedZeroTestsAndUnexpectedNonZeroExit_PrintsFailedSummary() + { + var capturingConsole = new CapturingConsole(); + var options = new TerminalTestReporterOptions + { + AllowZeroTests = true, + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + reporter.TestExecutionCompleted( + DateTimeOffset.UtcNow, + exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.GenericFailure); + + StripAnsi(capturingConsole.GetOutput()).Should().Contain("Test run summary: Failed!"); + } + /// /// When an assembly's tests were retried, the per-assembly summary should append a /// "/r{N}" segment to the compact counts block so users can tell the final counts came from retries. diff --git a/test/dotnet.Tests/CommandTests/Test/TestApplicationLaunchTests.cs b/test/dotnet.Tests/CommandTests/Test/TestApplicationLaunchTests.cs new file mode 100644 index 000000000000..df3b54a04b1e --- /dev/null +++ b/test/dotnet.Tests/CommandTests/Test/TestApplicationLaunchTests.cs @@ -0,0 +1,164 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Diagnostics; +using Microsoft.DotNet.Cli.Commands.Run; +using Microsoft.DotNet.Cli.Commands.Test; +using Microsoft.DotNet.Cli.Commands.Test.Terminal; +using Microsoft.DotNet.ProjectTools; +using Microsoft.Testing.Platform.OutputDevice.Terminal; + +namespace dotnet.Tests.CommandTests.Test; + +[TestClass] +public sealed class TestApplicationLaunchTests +{ + [TestMethod] + public void CreateProcessStartInfo_TopLevelAffectedTests_AddsOptionAndRunMarker() + { + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text) { AffectedTests = true }); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + startInfo.Arguments.Should().Contain("--affected-tests"); + startInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] + .Should().Be(TestOptions.RunAffectedTestsMode); + } + + [TestMethod] + public void CreateProcessStartInfo_TopLevelCollectTestMap_AddsOptionAndCollectMarker() + { + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text) { CollectTestMap = true }); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + startInfo.Arguments.Should().Contain("--collect-test-map"); + startInfo.Arguments.LastIndexOf("--collect-test-map", StringComparison.Ordinal) + .Should().Be(startInfo.Arguments.IndexOf("--collect-test-map", StringComparison.Ordinal)); + startInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] + .Should().Be(TestOptions.CollectTestMapMode); + } + + [TestMethod] + public void CreateProcessStartInfo_ForwardedAffectedTests_PreservesOriginalArgumentPosition() + { + string[] forwardedArguments = ["--minimum-expected-tests", "--affected-tests", "1"]; + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text) + { + AffectedTests = true, + AffectedTestsForwarded = true, + }, + forwardedArguments); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + int minimumIndex = startInfo.Arguments.IndexOf("--minimum-expected-tests", StringComparison.Ordinal); + int affectedIndex = startInfo.Arguments.IndexOf("--affected-tests", StringComparison.Ordinal); + int valueIndex = startInfo.Arguments.IndexOf(" 1", affectedIndex, StringComparison.Ordinal); + minimumIndex.Should().BeGreaterThanOrEqualTo(0); + affectedIndex.Should().BeGreaterThan(minimumIndex); + valueIndex.Should().BeGreaterThan(affectedIndex); + startInfo.Arguments.LastIndexOf("--affected-tests", StringComparison.Ordinal).Should().Be(affectedIndex); + startInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] + .Should().Be(TestOptions.RunAffectedTestsMode); + } + + [TestMethod] + public void CreateProcessStartInfo_ForwardedCollectTestMap_PreservesOriginalArgumentPosition() + { + string[] forwardedArguments = ["--filter", "TestClass", "--collect-test-map"]; + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text) + { + CollectTestMap = true, + CollectTestMapForwarded = true, + }, + forwardedArguments); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + int filterIndex = startInfo.Arguments.IndexOf("--filter", StringComparison.Ordinal); + int collectIndex = startInfo.Arguments.IndexOf("--collect-test-map", StringComparison.Ordinal); + filterIndex.Should().BeGreaterThanOrEqualTo(0); + collectIndex.Should().BeGreaterThan(filterIndex); + startInfo.Arguments.LastIndexOf("--collect-test-map", StringComparison.Ordinal).Should().Be(collectIndex); + startInfo.Environment[TestOptions.AffectedTestsModeEnvironmentVariable] + .Should().Be(TestOptions.CollectTestMapMode); + } + + [TestMethod] + public void CreateProcessStartInfo_OrdinaryRun_RemovesInheritedModuleMarker() + { + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text), + environmentVariables: new Dictionary + { + [TestOptions.AffectedTestsModeEnvironmentVariable] = TestOptions.CollectTestMapMode, + }); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + startInfo.Environment.ContainsKey(TestOptions.AffectedTestsModeEnvironmentVariable).Should().BeFalse(); + } + + [TestMethod] + public void CreateProcessStartInfo_LaunchProfileAffectedOption_DoesNotCreateSdkMarker() + { + var launchProfile = new ProjectLaunchProfile + { + CommandLineArgs = "--affected-tests", + EnvironmentVariables = ImmutableDictionary.Empty + .Add(TestOptions.AffectedTestsModeEnvironmentVariable, TestOptions.RunAffectedTestsMode), + }; + using TestApplication application = CreateApplication( + new TestOptions(false, false, TestListFormat.Text), + launchProfile: launchProfile); + + ProcessStartInfo startInfo = application.CreateProcessStartInfo(); + + startInfo.Arguments.Should().Contain("--affected-tests"); + startInfo.Environment.ContainsKey(TestOptions.AffectedTestsModeEnvironmentVariable).Should().BeFalse(); + } + + private static TestApplication CreateApplication( + TestOptions testOptions, + IEnumerable? forwardedArguments = null, + IReadOnlyDictionary? environmentVariables = null, + ProjectLaunchProfile? launchProfile = null) + { + var module = new TestModule( + new RunProperties("dotnet", "test.dll", null), + ProjectFullPath: "test.csproj", + TargetFramework: "net11.0", + IsTestingPlatformApplication: true, + LaunchSettings: launchProfile, + TargetPath: "test.dll", + DotnetRootArchVariableName: null, + EnvironmentVariables: environmentVariables ?? ImmutableDictionary.Empty); + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: forwardedArguments?.ToImmutableArray() ?? [], + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + var reporter = new TerminalTestReporter( + new CapturingConsole(), + new TerminalTestReporterOptions + { + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + }); + + return new TestApplication(module, buildOptions, testOptions, reporter, _ => { }); + } +} diff --git a/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs b/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs index 887b6349de9a..6dfb5052cb97 100644 --- a/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs @@ -1,9 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Immutable; using Microsoft.DotNet.Cli.Commands.Test; using Microsoft.DotNet.Cli.CommandLine; using Microsoft.DotNet.Cli.Extensions; +using Microsoft.DotNet.Cli.Utils; using TestCommand = Microsoft.DotNet.Cli.Commands.Test.TestCommand; namespace Microsoft.DotNet.Cli.Test.Tests @@ -277,6 +279,345 @@ public void MTPCommandAcceptsBareListTestsWithoutValue() parseResult.GetValue(command.ListTestsOption).Should().BeNull(); } + [TestMethod] + [DataRow("--collect-test-map")] + [DataRow("--affected-tests")] + public void MTPCommandAcceptsAffectedTestOptions(string option) + { + WithAffectedTestsFeature(enabled: true, () => + { + var command = new TestCommandDefinition.MicrosoftTestingPlatform(); + var parseResult = command.Parse([option]); + + parseResult.Errors.Should().BeEmpty(); + parseResult.HasOption( + option == "--collect-test-map" + ? command.CollectTestMapOption + : command.AffectedTestsOption).Should().BeTrue(); + }); + } + + [TestMethod] + public void MTPCommandRejectsAffectedTestOptionsTogether() + { + WithAffectedTestsFeature(enabled: true, () => + { + var command = new TestCommandDefinition.MicrosoftTestingPlatform(); + var parseResult = command.Parse(["--collect-test-map", "--affected-tests"]); + + parseResult.Errors.Should().ContainSingle() + .Which.Message.Should().Contain("cannot be used together"); + }); + } + + [TestMethod] + [DataRow("--collect-test-map")] + [DataRow("--affected-tests")] + public void MTPCommandRejectsAffectedTestOptionsWhenFeatureIsDisabled(string option) + { + WithAffectedTestsFeature(enabled: false, () => + { + var command = new TestCommandDefinition.MicrosoftTestingPlatform(); + var parseResult = command.Parse([option]); + + parseResult.Errors.Should().ContainSingle() + .Which.Message.Should().Contain(TestCommandDefinition.MicrosoftTestingPlatform.EnableAffectedTestsEnvironmentVariable); + command.CollectTestMapOption.Hidden.Should().BeTrue(); + command.AffectedTestsOption.Hidden.Should().BeTrue(); + }); + } + + [TestMethod] + public void MTPCommandRejectsCollectTestMapWithParallelModules() + { + WithAffectedTestsFeature(enabled: true, () => + { + var command = new TestCommandDefinition.MicrosoftTestingPlatform(); + var parseResult = command.Parse(["--collect-test-map", "--max-parallel-test-modules", "2"]); + + parseResult.Errors.Should().ContainSingle() + .Which.Message.Should().Contain("--max-parallel-test-modules"); + }); + } + + [TestMethod] + public void MTPCommandNormalizesAffectedOptionsForwardedAfterDoubleDash() + { + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: ImmutableArray.Create("--collect-test-map", "--other", "--affected-tests"), + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + + (BuildOptions normalized, bool collectTestMap, bool affectedTests) = + MicrosoftTestingPlatformTestCommand.NormalizeForwardedAffectedTestsOptions(buildOptions); + + collectTestMap.Should().BeTrue(); + affectedTests.Should().BeTrue(); + normalized.TestApplicationArguments.Should().Equal("--collect-test-map", "--other", "--affected-tests"); + } + + [DataRow("-affected-tests", true)] + [DataRow("--Affected-Tests", true)] + [DataRow("-AFFECTED-TESTS=true", false)] + [DataRow("---affected-tests", false)] + [DataRow("----affected-tests", false)] + [TestMethod] + public void MTPCommandNormalizesForwardedAffectedOptionSpellings(string option, bool expectedAffectedTests) + { + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: ImmutableArray.Create(option), + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + + (BuildOptions normalized, _, bool affectedTests) = + MicrosoftTestingPlatformTestCommand.NormalizeForwardedAffectedTestsOptions(buildOptions); + + affectedTests.Should().Be(expectedAffectedTests); + normalized.TestApplicationArguments.Should().Equal(option); + } + + [TestMethod] + public void MTPCommandDetectsAffectedOptionInForwardedResponseFile() + { + using var temp = new TempDirectory(); + string responseFile = Path.Combine(temp.Path, "affected.rsp"); + File.WriteAllText(responseFile, "--affected-tests"); + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: ImmutableArray.Create($"@{responseFile}"), + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + + (BuildOptions normalized, _, bool affectedTests) = + MicrosoftTestingPlatformTestCommand.NormalizeForwardedAffectedTestsOptions(buildOptions); + + affectedTests.Should().BeFalse(); + normalized.TestApplicationArguments.Should().Equal($"@{responseFile}"); + + (_, affectedTests, _) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + normalized.TestApplicationArguments, + [null], + Directory.GetCurrentDirectory()); + + affectedTests.Should().BeTrue(); + } + + [TestMethod] + public void MTPCommandDoesNotEnableAffectedTestsForValuedResponseFileOption() + { + using var temp = new TempDirectory(); + string responseFile = Path.Combine(temp.Path, "affected.rsp"); + File.WriteAllText(responseFile, "--affected-tests=false"); + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: ImmutableArray.Create($"@{responseFile}"), + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + + (_, _, bool affectedTests) = + MicrosoftTestingPlatformTestCommand.NormalizeForwardedAffectedTestsOptions(buildOptions); + + affectedTests.Should().BeFalse(); + + (_, affectedTests, _) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + buildOptions.TestApplicationArguments, + [null], + Directory.GetCurrentDirectory()); + + affectedTests.Should().BeFalse(); + } + + [TestMethod] + public void MTPCommandDetectsAffectedOptionInQuotedNestedResponseFile() + { + using var temp = new TempDirectory(); + string inner = Path.Combine(temp.Path, "inner.rsp"); + string outer = Path.Combine(temp.Path, "outer.rsp"); + File.WriteAllText(inner, "\"--affected-tests\""); + File.WriteAllText(outer, $"\"@{inner}\""); + var buildOptions = new BuildOptions( + new PathOptions(null, null, null, null, null, null), + HasNoRestore: false, + HasNoBuild: false, + Verbosity: null, + NoLaunchProfile: false, + NoLaunchProfileArguments: false, + TestApplicationArguments: ImmutableArray.Create($"@{outer}"), + MSBuildArgs: [], + Device: null, + ListDevices: false, + EnvironmentVariables: ImmutableDictionary.Empty); + + (_, _, bool affectedTests) = + MicrosoftTestingPlatformTestCommand.NormalizeForwardedAffectedTestsOptions(buildOptions); + + affectedTests.Should().BeFalse(); + + (_, affectedTests, _) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + buildOptions.TestApplicationArguments, + [null], + Directory.GetCurrentDirectory()); + + affectedTests.Should().BeTrue(); + } + + [TestMethod] + public void MTPCommandRejectsDifferentAffectedOperationsAcrossWorkingDirectories() + { + using var temp = new TempDirectory(); + string affectedDirectory = Path.Combine(temp.Path, "affected"); + string ordinaryDirectory = Path.Combine(temp.Path, "ordinary"); + Directory.CreateDirectory(affectedDirectory); + Directory.CreateDirectory(ordinaryDirectory); + File.WriteAllText(Path.Combine(affectedDirectory, "options.rsp"), "--affected-tests"); + File.WriteAllText(Path.Combine(ordinaryDirectory, "options.rsp"), "--filter TestClass"); + + Action action = () => + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray.Create("@options.rsp"), + [ordinaryDirectory, affectedDirectory], + temp.Path); + + action.Should().Throw() + .WithMessage("*same affected-test operation*"); + } + + [TestMethod] + public void MTPCommandMatchesMTPResponseFileQuoteBoundaries() + { + using var temp = new TempDirectory(); + string responseFile = Path.Combine(temp.Path, "affected.rsp"); + File.WriteAllText(responseFile, "\"--affected-tests\"\"--filter\""); + + (_, bool affectedTests, _) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray.Create("@affected.rsp"), + [null], + temp.Path); + + affectedTests.Should().BeTrue(); + } + + [TestMethod] + public void MTPCommandDoesNotPartiallyActivateMalformedResponseFile() + { + using var temp = new TempDirectory(); + string responseFile = Path.Combine(temp.Path, "affected.rsp"); + File.WriteAllLines(responseFile, ["--affected-tests", "--filter \"unclosed"]); + + (bool collectTestMap, bool affectedTests, bool minimumExpectedTests) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray.Create("@affected.rsp"), + [null], + temp.Path); + + collectTestMap.Should().BeFalse(); + affectedTests.Should().BeFalse(); + minimumExpectedTests.Should().BeFalse(); + } + + [TestMethod] + public void MTPCommandRejectsFeatureActivationWhenAnotherWorkingDirectoryCannotReadResponseFile() + { + using var temp = new TempDirectory(); + string affectedDirectory = Path.Combine(temp.Path, "affected"); + string missingDirectory = Path.Combine(temp.Path, "missing"); + Directory.CreateDirectory(affectedDirectory); + Directory.CreateDirectory(missingDirectory); + File.WriteAllText(Path.Combine(affectedDirectory, "options.rsp"), "--affected-tests"); + + Action action = () => + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray.Create("@options.rsp"), + [missingDirectory, affectedDirectory], + temp.Path); + + action.Should().Throw() + .WithMessage("*same affected-test operation*"); + } + + [TestMethod] + public void MTPCommandDetectsMinimumExpectedTestsInResponseFile() + { + using var temp = new TempDirectory(); + File.WriteAllText( + Path.Combine(temp.Path, "options.rsp"), + "--collect-test-map --minimum-expected-tests=1"); + + (bool collectTestMap, _, bool minimumExpectedTests) = + MicrosoftTestingPlatformTestCommand.DetectAffectedTestsOptionsInForwardedResponseFiles( + ImmutableArray.Create("@options.rsp"), + [null], + temp.Path); + + collectTestMap.Should().BeTrue(); + minimumExpectedTests.Should().BeTrue(); + } + + [TestMethod] + [DataRow(false, 0, 0, true)] + [DataRow(true, 0, 0, false)] + [DataRow(false, 2, 2, true)] + [DataRow(true, 2, 2, true)] + [DataRow(true, 2, 1, false)] + public void MTPCommandFailsOnlyForDisallowedEmptyOrAllSkippedRuns( + bool isAffectedTestsMode, + int totalTests, + int skippedTests, + bool expectedFailure) + { + MicrosoftTestingPlatformTestCommand.ShouldFailForNoExecutedTests( + isAffectedTestsMode, + totalTests, + skippedTests).Should().Be(expectedFailure); + } + + [TestMethod] + public void MTPCommandRejectsCollectTestMapWithMinimumExpectedTests() + { + WithAffectedTestsFeature(enabled: true, () => + { + var command = new TestCommandDefinition.MicrosoftTestingPlatform(); + var parseResult = command.Parse(["--collect-test-map", "--minimum-expected-tests", "1"]); + + parseResult.Errors.Should().ContainSingle() + .Which.Message.Should().Contain("--minimum-expected-tests"); + }); + } + [TestMethod] [DataRow("foo")] [DataRow("JSON")] @@ -368,6 +709,21 @@ public void Create_WhenGlobalJsonIsEmpty_FallsBackToVSTestInsteadOfThrowing() "an empty global.json must not crash the CLI parser (regression for https://github.com/dotnet/sdk/issues/52384)"); } + private static void WithAffectedTestsFeature(bool enabled, Action action) + { + const string variable = TestCommandDefinition.MicrosoftTestingPlatform.EnableAffectedTestsEnvironmentVariable; + string? previousValue = Environment.GetEnvironmentVariable(variable); + try + { + Environment.SetEnvironmentVariable(variable, enabled ? "1" : null); + action(); + } + finally + { + Environment.SetEnvironmentVariable(variable, previousValue); + } + } + [TestMethod] public void Create_WhenGlobalJsonIsMalformed_FallsBackToVSTestInsteadOfThrowing() {