diff options
21 files changed, 260 insertions, 60 deletions
diff --git a/Directory.Build.props b/Directory.Build.props index 5104b073389..ea12ca71919 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -93,6 +93,7 @@ <WorkloadBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WorkloadBuildTasksDir> <MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir> <MonoTargetsTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoTargetsTasksDir> + <TestExclusionListTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</TestExclusionListTasksDir> <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath> <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath> <Crossgen2SdkOverridePropsPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.props'))</Crossgen2SdkOverridePropsPath> @@ -104,6 +105,7 @@ <WorkloadBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll'))</WorkloadBuildTasksAssemblyPath> <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath> <MonoTargetsTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll'))</MonoTargetsTasksAssemblyPath> + <TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath> <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath> </PropertyGroup> diff --git a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml index 2e883dc9a6c..0868ffb1ac1 100644 --- a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml @@ -13,7 +13,6 @@ parameters: runtimeVariant: '' variables: {} pool: '' - runtimeFlavorDisplayName: 'Mono' dependsOn: [] #arcade-specific parameters condition: always() @@ -38,8 +37,8 @@ steps: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} runtimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(variables['System.TeamProject'], 'public') }}: diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index b1bc6a78bb3..d32ca3b6613 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -11,7 +11,6 @@ parameters: variables: {} pool: '' runtimeFlavor: 'coreclr' - runtimeFlavorDisplayName: 'CoreCLR' runtimeVariant: '' dependsOn: [] dependOnEvaluatePaths: false @@ -52,10 +51,10 @@ jobs: # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' # Since the condition is being altered, merge the default with the additional conditions. # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 03288003d85..c553827ab39 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -21,7 +21,6 @@ parameters: variables: {} pool: '' runtimeFlavor: 'coreclr' - runtimeFlavorDisplayName: 'CoreCLR' shouldContinueOnError: false dependsOn: [] dependOnEvaluatePaths: false @@ -81,10 +80,10 @@ jobs: # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: 'run_test_p0_${{ parameters.runtimeFlavor }}${{ parameters.runtimeVariant }}_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' variables: @@ -335,7 +334,7 @@ jobs: archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} runtimeVariant: ${{ parameters.runtimeVariant }} diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index fe67ff83772..99a3de5196a 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -24,7 +24,7 @@ parameters: tieringTest: '' longRunningGcTests: '' gcSimulatorTests: '' - runtimeFlavorDisplayName: 'CoreCLR' + runtimeFlavor: 'CoreCLR' runtimeVariant: '' shouldContinueOnError: false @@ -57,7 +57,7 @@ steps: _PALTestsDir: ${{ parameters.runPALTestsDir }} _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + RuntimeFlavor: ${{ parameters.runtimeFlavor }} _RuntimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(parameters.publishTestResults, 'true') }}: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml index 6dd5f6b858b..f1b3bbe955a 100644 --- a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml @@ -18,8 +18,6 @@ parameters: runtimeVariant: '' variables: {} pool: '' - runtimeFlavor: 'mono' - runtimeFlavorDisplayName: 'Mono' dependsOn: [] #arcade-specific parameters condition: always() @@ -32,7 +30,7 @@ parameters: steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=monointerpreter /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper) displayName: Build Tests # Send tests to Helix @@ -44,8 +42,9 @@ steps: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} + runtimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(variables['System.TeamProject'], 'public') }}: creator: $(Build.DefinitionName) diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml index aa7e4257764..e6bb45d9888 100644 --- a/eng/pipelines/common/templates/wasm-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasm-runtime-tests.yml @@ -28,6 +28,7 @@ jobs: testGroup: innerloop isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} nameSuffix: AllSubsets_Mono_RuntimeTests + runtimeVariant: monointerpreter buildArgs: -s mono+libs -c $(_BuildConfig) timeoutInMinutes: 180 condition: >- diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 6ea24574d3a..e6ac905afb0 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -14,11 +14,6 @@ parameters: jobs: - template: ${{ coalesce(parameters.helixQueuesTemplate, parameters.jobTemplate) }} parameters: - ${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}: - runtimeFlavorDisplayName: 'CoreCLR' - ${{ if eq(parameters.jobParameters.runtimeFlavor, 'mono') }}: - runtimeFlavorDisplayName: 'Mono' - shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }} # keep in sync with /eng/pipelines/common/variables.yml diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 45ac52c4dcd..43b78cee145 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -7,7 +7,6 @@ parameters: container: '' pool: '' platform: '' - runtimeFlavorDisplayName: '' shouldContinueOnError: false dependOnEvaluatePaths: false jobParameters: {} @@ -23,8 +22,7 @@ jobs: pool: ${{ parameters.pool }} platform: ${{ parameters.platform }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} helixQueues: # iOS/tvOS simulator x64/x86 diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index d67e4cec123..ecca458ba59 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -7,7 +7,6 @@ parameters: container: '' pool: '' platform: '' - runtimeFlavorDisplayName: '' shouldContinueOnError: false dependOnEvaluatePaths: false jobParameters: {} @@ -24,7 +23,6 @@ jobs: platform: ${{ parameters.platform }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} helixQueues: # Linux arm diff --git a/src/tasks/AndroidAppBuilder/ApkBuilder.cs b/src/tasks/AndroidAppBuilder/ApkBuilder.cs index b6acdc48592..2d54e5192f8 100644 --- a/src/tasks/AndroidAppBuilder/ApkBuilder.cs +++ b/src/tasks/AndroidAppBuilder/ApkBuilder.cs @@ -463,7 +463,7 @@ public class ApkBuilder // 4. Align APK string alignedApk = Path.Combine(OutputDir, "bin", $"{ProjectName}.apk"); - Utils.RunProcess(logger, zipalign, $"-v 4 {apkFile} {alignedApk}", workingDir: OutputDir); + AlignApk(apkFile, alignedApk, zipalign); // we don't need the unaligned one any more File.Delete(apkFile); @@ -475,6 +475,11 @@ public class ApkBuilder return (alignedApk, packageId); } + private void AlignApk(string unalignedApkPath, string apkOutPath, string zipalign) + { + Utils.RunProcess(logger, zipalign, $"-v 4 {unalignedApkPath} {apkOutPath}", workingDir: OutputDir); + } + private void SignApk(string apkPath, string apksigner) { string defaultKey = Path.Combine(OutputDir, "debug.keystore"); @@ -495,6 +500,31 @@ public class ApkBuilder $"--ks-pass pass:android --key-pass pass:android {apkPath}", workingDir: OutputDir); } + public void ZipAndSignApk(string apkPath) + { + if (string.IsNullOrEmpty(AndroidSdk)) + AndroidSdk = Environment.GetEnvironmentVariable("ANDROID_SDK_ROOT"); + + if (string.IsNullOrEmpty(AndroidSdk) || !Directory.Exists(AndroidSdk)) + throw new ArgumentException($"Android SDK='{AndroidSdk}' was not found or incorrect (can be set via ANDROID_SDK_ROOT envvar)."); + + if (string.IsNullOrEmpty(BuildToolsVersion)) + BuildToolsVersion = GetLatestBuildTools(AndroidSdk); + + if (string.IsNullOrEmpty(MinApiLevel)) + MinApiLevel = DefaultMinApiLevel; + + string buildToolsFolder = Path.Combine(AndroidSdk, "build-tools", BuildToolsVersion); + string zipalign = Path.Combine(buildToolsFolder, "zipalign"); + string apksigner = Path.Combine(buildToolsFolder, "apksigner"); + + string alignedApkPath = $"{apkPath}.aligned"; + AlignApk(apkPath, alignedApkPath, zipalign); + logger.LogMessage(MessageImportance.High, $"\nMoving '{alignedApkPath}' to '{apkPath}'.\n"); + File.Move(alignedApkPath, apkPath, overwrite: true); + SignApk(apkPath, apksigner); + } + public void ReplaceFileInApk(string file) { if (string.IsNullOrEmpty(AndroidSdk)) diff --git a/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs b/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs new file mode 100644 index 00000000000..254bef87ef6 --- /dev/null +++ b/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +namespace TestExclusionListTasks; + +public class PatchExclusionListInApks : Task +{ + [Required] + public ITaskItem[]? ApkPaths { get; set; } + + [Required] + public ITaskItem[]? ExcludedTests { get; set; } + + public string? AndroidSdk { get; set; } + + public string? MinApiLevel { get; set; } + + public string? BuildToolsVersion { get; set; } + + public string? KeyStorePath { get; set; } + + public override bool Execute() + { + var apkBuilder = new ApkBuilder(Log); + apkBuilder.AndroidSdk = AndroidSdk; + apkBuilder.MinApiLevel = MinApiLevel; + apkBuilder.BuildToolsVersion = BuildToolsVersion; + apkBuilder.KeyStorePath = KeyStorePath; + + string testExclusionList = string.Join( + '\n', + (ExcludedTests ?? Enumerable.Empty<ITaskItem>()).Select(t => t.ItemSpec)); + foreach (ITaskItem apk in ApkPaths ?? Enumerable.Empty<ITaskItem>()) + { + string apkPath = apk.GetMetadata("FullPath")!; + apkBuilder.OutputDir = Path.GetDirectoryName(apkPath)!; + using (ZipArchive apkArchive = ZipFile.Open(apkPath, ZipArchiveMode.Update)) + { + ZipArchiveEntry assetsZipEntry = apkArchive.GetEntry("assets/assets.zip")!; + using ZipArchive assetsArchive = new ZipArchive(assetsZipEntry.Open(), ZipArchiveMode.Update); + ZipArchiveEntry testExclusionListEntry = assetsArchive.GetEntry("TestExclusionList.txt")!; + using StreamWriter textExclusionListWriter = new StreamWriter(testExclusionListEntry.Open()); + textExclusionListWriter.WriteLine(testExclusionList); + } + apkBuilder.ZipAndSignApk(apkPath); + } + return true; + } +} diff --git a/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj b/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj new file mode 100644 index 00000000000..d17dc95c41a --- /dev/null +++ b/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj @@ -0,0 +1,19 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>$(TargetFrameworkForNETCoreTasks)</TargetFramework> + <OutputType>Library</OutputType> + <TreatWarningsAsErrors>true</TreatWarningsAsErrors> + <EnableDefaultCompileItems>false</EnableDefaultCompileItems> + <Nullable>enable</Nullable> + <NoWarn>$(NoWarn),CA1050</NoWarn> + </PropertyGroup> + + <ItemGroup> + <Compile Include="PatchExclusionListInApks.cs" /> + <Compile Include="../AndroidAppBuilder/ApkBuilder.cs" /> + <Compile Include="../Common/Utils.cs" /> + </ItemGroup> + <ItemGroup> + <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" /> + </ItemGroup> +</Project> diff --git a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs index 89fafa77da4..976f542a6b2 100644 --- a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs +++ b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs @@ -42,6 +42,12 @@ namespace TestLibrary } } + public static bool OutOfProcessTestsSupported => + !OperatingSystem.IsIOS() + && !OperatingSystem.IsTvOS() + && !OperatingSystem.IsAndroid() + && !OperatingSystem.IsBrowser(); + public static void RunOutOfProcessTest(string basePath, string assemblyPath) { int ret = -100; diff --git a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs index b1b1f4e44e7..daf22773002 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs @@ -16,6 +16,11 @@ public static class RunnerEntryPoint string? filter, HashSet<string> testExclusionList) { + // If an exclusion list is passed as a filter, treat it as though no filter is provided here. + if (filter?.StartsWith("--exclusion-list=") == true) + { + filter = null; + } ApplicationEntryPoint? entryPoint = null; if (OperatingSystem.IsAndroid()) { diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 6559a6db44f..d0db16db340 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -249,7 +249,12 @@ sealed class OutOfProcessTest : ITestInfo Method = displayName; DisplayNameForFiltering = displayName; TestNameExpression = $"@\"{displayName}\""; - ExecutionStatement = $@"TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");"; + ExecutionStatement = $@" +if (TestLibrary.OutOfProcessTest.OutOfProcessTestsSupported) +{{ +TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}""); +}} +"; } public string TestNameExpression { get; } diff --git a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs index a2cf5f0c704..6c4c4f6b57b 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; namespace XUnitWrapperLibrary; @@ -47,6 +48,11 @@ public class TestFilter } return stringToSearch == Filter; } + + public override string ToString() + { + return $"{Kind}{(Substring ? "~" : "=")}{Filter}"; + } } public sealed class AndClause : ISearchClause @@ -61,6 +67,11 @@ public class TestFilter } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => _left.IsMatch(fullyQualifiedName, displayName, traits) && _right.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"({_left}) && ({_right})"; + } } public sealed class OrClause : ISearchClause @@ -75,6 +86,11 @@ public class TestFilter } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => _left.IsMatch(fullyQualifiedName, displayName, traits) || _right.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"({_left}) || ({_right})"; + } } public sealed class NotClause : ISearchClause @@ -87,6 +103,11 @@ public class TestFilter } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => !_inner.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"!({_inner})"; + } } private readonly ISearchClause? _filter; @@ -133,6 +154,17 @@ public class TestFilter public static HashSet<string> LoadTestExclusionList() { HashSet<string> output = new (); + + // Try reading the exclusion list as a base64-encoded semicolon-delimited string as a commmand-line arg. + string[] arguments = Environment.GetCommandLineArgs(); + string? testExclusionListArg = arguments.FirstOrDefault(arg => arg.StartsWith("--exclusion-list=")); + if (testExclusionListArg is not null) + { + string testExclusionListPathFromCommandLine = testExclusionListArg.Substring("--exclusion-list=".Length); + output.UnionWith(File.ReadAllLines(testExclusionListPathFromCommandLine)); + } + + // Try reading the exclusion list as a line-delimited file. string? testExclusionListPath = Environment.GetEnvironmentVariable("TestExclusionListPath"); if (!string.IsNullOrEmpty(testExclusionListPath)) { diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 4bf0fbd0604..ccb695afd9c 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -69,7 +69,7 @@ <Delete Files="@(CleanPreexistingPayloads)" /> <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" /> - <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadDirectories" Properties="Scenarios=$(_Scenarios)" StopOnFirstFailure="true" /> + <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadDirectories" Properties="$(_PropertiesToPass)" StopOnFirstFailure="true" /> <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePALTestArchive" Properties="$(_PropertiesToPass)" StopOnFirstFailure="true" /> <ItemGroup> @@ -324,14 +324,36 @@ <ReducedMergedPayloadFilesFinal Include="@(_MergedPayloadFiles)" Condition=" '%(Extension)' != '.pdb' and '%(Extension)' != '.OutOfProcessTest' " /> </ItemGroup> + <ItemGroup> + <!-- We need to ensure that the test run script is marked as executable. --> + <HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true'" Include="chmod +x $(_MergedWrapperRunScriptRelative)" /> + <!-- Force assemblies to lazy-load for LLVM AOT test runs to enable using tests that fail at AOT time (and as a result can't be AOTd) --> + <HelixCommandLines Condition="'$(RuntimeVariant)' == 'llvmfullaot'" Include="$(_MergedWrapperRunScriptRelative) --aot-lazy-assembly-load" /> + <HelixCommandLines Condition="'$(RuntimeVariant)' != 'llvmfullaot'" Include="$(_MergedWrapperRunScriptRelative)" /> + </ItemGroup> + <Copy SourceFiles="@(ReducedMergedPayloadFilesFinal)" DestinationFiles="@(ReducedMergedPayloadFilesFinal->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" /> - <WriteLinesToFile File="$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\HelixCommand.txt" Lines="$(_MergedWrapperRunScriptRelative)" /> + <WriteLinesToFile File="$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\HelixCommand.txt" Lines="@(HelixCommandLines)" /> + </Target> + + <Target Name="GetTestExclusionList" Returns="@(FilteredTestExclusionList)" Condition="'$(TargetsMobile)' == 'true'"> + <ItemGroup> + <_ExcludeListProject Include="$(RepoRoot)src/tests/issues.targets"> + <AdditionalProperties>XunitTestBinBase=$(TestBinDir);$(_PropertiesToPass)</AdditionalProperties> + </_ExcludeListProject> + </ItemGroup> + <MSBuild Projects="@(_ExcludeListProject)" + Targets="GetFilteredExcludeList"> + <Output TaskParameter="TargetOutputs" ItemName="FilteredTestExclusionList" /> + </MSBuild> </Target> + <UsingTask TaskName="PatchExclusionListInApks" AssemblyFile="$(TestExclusionListTasksAssemblyPath)" /> + <Target Name="PrepareMergedTestPayloadDirectoryForAndroid" Condition="'$(TargetsAndroid)' == 'true'" Outputs="%(_MergedWrapperMarker.FileName)" - DependsOnTargets="DiscoverMergedTestWrappers"> + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <PropertyGroup> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory)</_MergedWrapperDirectory> <_MergedWrapperName>%(_MergedWrapperMarker.FileName)</_MergedWrapperName> @@ -343,12 +365,15 @@ </ItemGroup> <Copy SourceFiles="@(_MergedPayloadFiles)" DestinationFiles="@(_MergedPayloadFiles->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileName)%(Extension)')" /> + <PatchExclusionListInApks + ApkPaths="@(_MergedPayloadFiles->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileName)%(Extension)')" + ExcludedTests="@(FilteredTestExclusionList)" /> </Target> <Target Name="PrepareMergedTestPayloadDirectoryForWasm" Condition="'$(TargetsBrowser)' == 'true'" Outputs="%(_MergedWrapperMarker.FileName)" - DependsOnTargets="DiscoverMergedTestWrappers"> + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <ItemGroup> <_MergedWrapperRunScript Include="@(_MergedWrapperMarker->'%(RootDir)%(Directory)AppBundle/RunTests.$(TestScriptExtension)')" /> </ItemGroup> @@ -367,16 +392,29 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %<xx>. --> <FileRelativeToPayloadsRootDirectory>$([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle', %(FullPath)))</FileRelativeToPayloadsRootDirectory> </_MergedPayloadFiles> + <!-- Remove the placeholder test exclusion list here --> + <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> + <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> + </ItemGroup> + + <ItemGroup> + <HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true'" Include="export WasmTestAppArgs="$WasmTestAppArgs --exclusion-list=TestExclusionList.txt"" /> + <HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' == 'true'" Include="set WasmTestAppArgs=%WasmTestAppArgs% --exclusion-list=TestExclusionList.txt" /> + <HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true'" Include="chmod +x $(_MergedWrapperRunScriptRelative)" /> + <HelixCommandLines Include="$(_MergedWrapperRunScriptRelative)" /> </ItemGroup> <Copy SourceFiles="@(_MergedPayloadFiles)" DestinationFiles="@(_MergedPayloadFiles->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" /> - <WriteLinesToFile File="$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\HelixCommand.txt" Lines="$(_MergedWrapperRunScriptRelative)" /> + <WriteLinesToFile File="$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\HelixCommand.txt" Lines="@(HelixCommandLines)" /> + <!-- Write the real test exclusion list instead of the placeholder. --> + <WriteLinesToFile File="@(_TestExclusionListPlaceholder->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" + Lines="@(FilteredTestExclusionList)" /> </Target> <Target Name="PrepareMergedTestPayloadDirectoryForAppleMobile" Condition="'$(TargetsAppleMobile)' == 'true'" Outputs="%(_MergedWrapperMarker.FileName)" - DependsOnTargets="DiscoverMergedTestWrappers"> + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <PropertyGroup> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory)</_MergedWrapperDirectory> <_MergedWrapperName>%(_MergedWrapperMarker.FileName)</_MergedWrapperName> @@ -400,10 +438,16 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %<xx>. --> <FileRelativeToPayloadsRootDirectory>$([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle/$(_MergedWrapperName)/$(Configuration)-$(XCodeSdk)', %(FullPath)))</FileRelativeToPayloadsRootDirectory> </_MergedPayloadFiles> + <!-- Remove the placeholder test exclusion list here --> + <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> + <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> </ItemGroup> <Copy SourceFiles="@(_MergedPayloadFiles)" DestinationFiles="@(_MergedPayloadFiles->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" /> - </Target> + <!-- Write the real test exclusion list instead of the placeholder. --> + <WriteLinesToFile File="@(_TestExclusionListPlaceholder->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" + Lines="@(FilteredTestExclusionList)" /> + </Target> <Target Name="PrepareMergedTestPayloadDirectory" DependsOnTargets="DiscoverMergedTestWrappers;PrepareMergedTestPayloadDirectoryForDesktop;PrepareMergedTestPayloadDirectoryForAndroid;PrepareMergedTestPayloadDirectoryForWasm;PrepareMergedTestPayloadDirectoryForAppleMobile" /> @@ -416,7 +460,10 @@ </LegacyPayloads> </ItemGroup> - <ZipDirectory SourceDirectory="@(LegacyPayloads->Metadata('PayloadDirectory'))" DestinationFile="$(LegacyPayloadsRootDirectory)\%(LegacyPayloads.PayloadGroup).zip" /> + <ZipDirectory + Condition="'@(LegacyPayloads)' != ''" + SourceDirectory="@(LegacyPayloads->Metadata('PayloadDirectory'))" + DestinationFile="$(LegacyPayloadsRootDirectory)\%(LegacyPayloads.PayloadGroup).zip" /> </Target> <Target Name="ZipMergedPayloadDirectory" AfterTargets="PrepareMergedTestPayloadDirectories" Condition="'$(TargetsBrowser)' == 'true'"> @@ -428,7 +475,10 @@ </MergedPayloads> </ItemGroup> - <ZipDirectory SourceDirectory="@(MergedPayloads->Metadata('PayloadDirectory'))" DestinationFile="$(MergedPayloadsRootDirectory)\%(MergedPayloads.PayloadGroup).zip" /> + <ZipDirectory + Condition="'@(MergedPayloads)' != ''" + SourceDirectory="@(MergedPayloads->Metadata('PayloadDirectory'))" + DestinationFile="$(MergedPayloadsRootDirectory)\%(MergedPayloads.PayloadGroup).zip" /> </Target> <PropertyGroup> @@ -443,7 +493,7 @@ <RunCrossGen2 Condition=" '$(RunCrossGen2)' != 'true' ">false</RunCrossGen2> <LongRunningGCTests Condition=" '$(LongRunningGCTests)' != 'true' ">false</LongRunningGCTests> <GcSimulatorTests Condition=" '$(GcSimulatorTests)' != 'true' ">false</GcSimulatorTests> - <TestRunNamePrefix Condition="'$(RuntimeFlavorDisplayName)' != ''">$(RuntimeFlavorDisplayName) </TestRunNamePrefix> + <TestRunNamePrefix Condition="'$(RuntimeFlavor)' != ''">$(RuntimeFlavor) </TestRunNamePrefix> <TestRunNamePrefix Condition=" '$(RunCrossGen)' == 'true' ">R2R </TestRunNamePrefix> <TestRunNamePrefix Condition=" '$(RunCrossGen2)' == 'true' ">R2R-CG2 </TestRunNamePrefix> <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(TargetOS) $(TargetArchitecture) $(Configuration) @ </TestRunNamePrefix> @@ -542,16 +592,7 @@ </MergedPayloads> </ItemGroup> - <ItemGroup Condition="'$(TargetsMobile)' != 'true'"> - <MergedPayloads Update="@(MergedPayloads)"> - <PayloadHelixCommandFile>$(MergedPayloadsRootDirectory)%(FileName)%(Extension)\HelixCommand.txt</PayloadHelixCommandFile> - <MergedTestHelixCommand>$([System.IO.File]::ReadAllText('%(PayloadHelixCommandFile)'))</MergedTestHelixCommand> - <PayloadGroup>$([System.IO.Path]::GetDirectoryName('%(MergedTestHelixCommand)').Replace('/', '-').Replace('\', '-'))</PayloadGroup> - <PayloadZipFile>$(MergedPayloadsRootDirectory)%(PayloadGroup).zip</PayloadZipFile> - </MergedPayloads> - </ItemGroup> - - <ItemGroup Condition="'$(TargetsBrowser)' == 'true'"> + <ItemGroup Condition="'$(TargetHasHelixXHarnessSdkSupport)' != 'true'"> <MergedPayloads Update="@(MergedPayloads)"> <PayloadHelixCommandFile>$(MergedPayloadsRootDirectory)%(FileName)%(Extension)\HelixCommand.txt</PayloadHelixCommandFile> <MergedTestHelixCommand>$([System.IO.File]::ReadAllText('%(PayloadHelixCommandFile)'))</MergedTestHelixCommand> @@ -560,12 +601,6 @@ </MergedPayloads> </ItemGroup> - <ItemGroup Condition="'$(TargetHasHelixXHarnessSdkSupport)' != 'true'"> - <MergedPayloads Update="@(MergedPayloads)" Condition="'$(TestWrapperTargetsWindows)' != 'true'"> - <MergedTestHelixCommand>chmod +x %(MergedPayloads.MergedTestHelixCommand)%0A%(MergedPayloads.MergedTestHelixCommand)</MergedTestHelixCommand> - </MergedPayloads> - </ItemGroup> - <ItemGroup Condition="'$(TargetHasHelixXHarnessSdkSupport)' == 'true'"> <MergedPayloads Update="@(MergedPayloads)"> <PayloadGroup>%(FileName)%(Extension)</PayloadGroup> @@ -595,6 +630,7 @@ </XHarnessApkToTest> <XHarnessApkToTest Include="@(MergedPayloads->Metadata('PayloadGroup'))" Condition="'$(TargetsAndroid)' == 'true'"> + <Arguments>--arg=env:TestExclusionListPath=TestExclusionList.txt</Arguments> <AndroidPackageName>net.dot.%(PayloadGroup)</AndroidPackageName> <AndroidInstrumentationName>net.dot.MonoRunner</AndroidInstrumentationName> <TestTimeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</TestTimeout> @@ -609,6 +645,7 @@ </XHarnessAppBundleToTest> <XHarnessAppBundleToTest Include="@(MergedPayloads->Metadata('PayloadGroup'))" Condition="'$(TargetsAppleMobile)' == 'true'"> + <Arguments>--set-env=TestExclusionListPath=TestExclusionList.txt</Arguments> <TestTarget Condition="'$(TargetArchitecture)' == 'arm64'">ios-simulator-64</TestTarget> <TestTarget Condition="'$(TargetArchitecture)' == 'x64'">ios-simulator-64</TestTarget> <TestTimeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</TestTimeout> diff --git a/src/tests/Common/mergedrunnermobile.targets b/src/tests/Common/mergedrunnermobile.targets index f595bb35745..7371cdc702c 100644 --- a/src/tests/Common/mergedrunnermobile.targets +++ b/src/tests/Common/mergedrunnermobile.targets @@ -21,6 +21,24 @@ <ProjectReference Include="$(RepoRoot)/src/tests/Common/XHarnessRunnerLibrary/XHarnessRunnerLibrary.csproj" /> </ItemGroup> + <!-- + We can't publish the test exclusion list at test build time as we don't know the full test run configuration + until we're about to send the test runs to Helix. Here we'll include a dummy text exclusion list into the app bundle + that we will patch with the actual test exclusion list later. + Adding the exclusion list here allows us to avoid having to manually patch the various metadata files for each app type + and allows us to only have to replace this file. + --> + <Target Name="_AddDummyTestExclusionListToAssets" BeforeTargets="AssignTargetPaths"> + <ItemGroup> + <_PlaceholderExclusionList Include="$(IntermediateOutputPath)/PlaceholderTestExclusionList.txt" /> + <Content Include="@(_PlaceholderExclusionList)" TargetPath="TestExclusionList.txt" CopyToOutputDirectory="PreserveNewest"/> + </ItemGroup> + <WriteLinesToFile File="@(_PlaceholderExclusionList)" + Lines="" + Overwrite="true" + WriteOnlyWhenDifferent="true" /> + </Target> + <!-- Don't use the usual script generation, the scripts it generates are not useful for these tests. --> <Target Name="GenerateExecutionScriptsInternal" /> <Target Name="GenerateLocalRunnerScript" DependsOnTargets="GenerateRunScript" AfterTargets="PublishTestAsSelfContained" /> diff --git a/src/tests/build.proj b/src/tests/build.proj index c292d42fe4c..78700b40b7c 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -597,17 +597,11 @@ Properties="@(CreateLibProperty)" /> </Target> - <Target Name="EmitTestExclusionList"> + <Target Name="EmitTestExclusionList" DependsOnTargets="GetFilteredExcludeList"> <PropertyGroup> <TestExclusionListPath>$(CORE_ROOT)\TestExclusionList.txt</TestExclusionListPath> </PropertyGroup> - <ItemGroup> - <FilteredExcludeList - Include="$([System.IO.Path]::GetRelativePath('$(XunitTestBinBase)', '%(ExcludeList.FullPath)'))" - Condition="'%(ExcludeList.Extension)' == '.dll'" /> - </ItemGroup> - <WriteLinesToFile File="$(TestExclusionListPath)" Lines="@(FilteredExcludeList)" diff --git a/src/tests/issues.targets b/src/tests/issues.targets index f8c0d3b91a5..caaae135a4f 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -4001,4 +4001,12 @@ <Issue>mobile and wasm don't support tests with native libraries. wasm also needs static linking</Issue> </ExcludeList> </ItemGroup> + + <Target Name="GetFilteredExcludeList" Returns="@(FilteredExcludeList)"> + <ItemGroup> + <FilteredExcludeList + Include="$([System.IO.Path]::GetRelativePath('$(XunitTestBinBase)', '%(ExcludeList.FullPath)'))" + Condition="'%(ExcludeList.Extension)' == '.dll'" /> + </ItemGroup> + </Target> </Project> |