diff options
author | Ankit Jain <radical@gmail.com> | 2021-07-14 17:47:03 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-07-14 17:47:03 +0300 |
commit | d953229d5429d2ffde833740dd481aab864d3e0c (patch) | |
tree | 74d4105746511db61bdd80f69a010bd057e67fb8 /src/tasks | |
parent | 9ad44f8347be52df4070d85264bfe528f34ba3a0 (diff) |
[wasm] Run `Wasm.Build.Tests` against workloads (#54451)
Co-authored-by: Larry Ewing <lewing@microsoft.com>
Diffstat (limited to 'src/tasks')
-rw-r--r-- | src/tasks/Common/Utils.cs | 4 | ||||
-rw-r--r-- | src/tasks/WasmAppBuilder/EmccCompile.cs | 4 | ||||
-rw-r--r-- | src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs | 228 | ||||
-rw-r--r-- | src/tasks/WorkloadBuildTasks/PackageInstaller.cs | 163 | ||||
-rw-r--r-- | src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj | 2 |
5 files changed, 397 insertions, 4 deletions
diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index ea2aba607dd..7b25107d56f 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -185,12 +185,12 @@ internal static class Utils } #if NETCOREAPP - public static void DirectoryCopy(string sourceDir, string destDir, Func<string, bool> predicate) + public static void DirectoryCopy(string sourceDir, string destDir, Func<string, bool>? predicate=null) { string[] files = Directory.GetFiles(sourceDir, "*", SearchOption.AllDirectories); foreach (string file in files) { - if (!predicate(file)) + if (predicate != null && !predicate(file)) continue; string relativePath = Path.GetRelativePath(sourceDir, file); diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index 78a6a4b1faa..2a8a2f12cbb 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -123,7 +123,7 @@ namespace Microsoft.WebAssembly.Build.Tasks if (exitCode != 0) { - Log.LogError($"Failed to compile {srcFile} -> {objFile}: {output}"); + Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output}"); return false; } @@ -135,7 +135,7 @@ namespace Microsoft.WebAssembly.Build.Tasks } catch (Exception ex) { - Log.LogError($"Failed to compile {srcFile} -> {objFile}: {ex.Message}"); + Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}"); return false; } } diff --git a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs new file mode 100644 index 00000000000..c76659d5db3 --- /dev/null +++ b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs @@ -0,0 +1,228 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +#nullable enable + +namespace Microsoft.Workload.Build.Tasks +{ + public class InstallWorkloadFromArtifacts : Task + { + [Required, NotNull] + public ITaskItem? WorkloadId { get; set; } + + [Required, NotNull] + public string? VersionBand { get; set; } + + [Required, NotNull] + public string? LocalNuGetsPath { get; set; } + + [Required, NotNull] + public string? SdkDir { get; set; } + + public ITaskItem[] ExtraNuGetSources { get; set; } = Array.Empty<ITaskItem>(); + + public override bool Execute() + { + Utils.Logger = Log; + + if (!HasMetadata(WorkloadId, nameof(WorkloadId), "Version") || + !HasMetadata(WorkloadId, nameof(WorkloadId), "ManifestName")) + { + return false; + } + + if (!Directory.Exists(SdkDir)) + { + Log.LogError($"Cannot find SdkDir={SdkDir}"); + return false; + } + + Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Installing workload manifest {WorkloadId.ItemSpec} **{Environment.NewLine}"); + + string nugetConfigContents = GetNuGetConfig(); + if (!InstallWorkloadManifest(WorkloadId.GetMetadata("ManifestName"), WorkloadId.GetMetadata("Version"), nugetConfigContents, stopOnMissing: true)) + return false; + + string nugetConfigPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + File.WriteAllText(nugetConfigPath, nugetConfigContents); + + Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** workload install **{Environment.NewLine}"); + (int exitCode, string output) = Utils.TryRunProcess( + Path.Combine(SdkDir, "dotnet"), + $"workload install --skip-manifest-update --no-cache --configfile \"{nugetConfigPath}\" {WorkloadId.ItemSpec}", + workingDir: Path.GetTempPath(), + silent: false, + debugMessageImportance: MessageImportance.High); + if (exitCode != 0) + { + Log.LogError($"workload install failed: {output}"); + + foreach (var dir in Directory.EnumerateDirectories(Path.Combine(SdkDir, "sdk-manifests"), "*", SearchOption.AllDirectories)) + Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(SdkDir, "sdk-manifests", dir)}"); + + foreach (var dir in Directory.EnumerateDirectories(Path.Combine(SdkDir, "packs"), "*", SearchOption.AllDirectories)) + Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(SdkDir, "packs", dir)}"); + + return false; + } + + return !Log.HasLoggedErrors; + } + + private string GetNuGetConfig() + { + StringBuilder nugetConfigBuilder = new(); + nugetConfigBuilder.AppendLine($"<configuration>{Environment.NewLine}<packageSources>"); + + nugetConfigBuilder.AppendLine($@"<add key=""nuget-local"" value=""{LocalNuGetsPath}"" />"); + foreach (ITaskItem source in ExtraNuGetSources) + { + string key = source.ItemSpec; + string value = source.GetMetadata("Value"); + if (string.IsNullOrEmpty(value)) + { + Log.LogWarning($"ExtraNuGetSource {key} is missing Value metadata"); + continue; + } + + nugetConfigBuilder.AppendLine($@"<add key=""{key}"" value=""{value}"" />"); + } + + nugetConfigBuilder.AppendLine($"</packageSources>{Environment.NewLine}</configuration>"); + return nugetConfigBuilder.ToString(); + } + + private bool InstallWorkloadManifest(string name, string version, string nugetConfigContents, bool stopOnMissing) + { + Log.LogMessage(MessageImportance.High, $"Installing workload manifest for {name}/{version}"); + + // Find any existing directory with the manifest name, ignoring the case + // Multiple directories for a manifest, differing only in case causes + // workload install to fail due to duplicate manifests! + // This is applicable only on case-sensitive filesystems + string outputDir = FindSubDirIgnoringCase(Path.Combine(SdkDir, "sdk-manifests", VersionBand), name); + + PackageReference pkgRef = new(Name: $"{name}.Manifest-{VersionBand}", + Version: version, + OutputDir: outputDir, + relativeSourceDir: "data"); + + if (!PackageInstaller.Install(new[]{ pkgRef }, nugetConfigContents, Log, stopOnMissing)) + return false; + + string manifestDir = pkgRef.OutputDir; + string jsonPath = Path.Combine(manifestDir, "WorkloadManifest.json"); + if (!File.Exists(jsonPath)) + { + Log.LogError($"Could not find WorkloadManifest.json at {jsonPath}"); + return false; + } + + ManifestInformation? manifest; + try + { + manifest = JsonSerializer.Deserialize<ManifestInformation>( + File.ReadAllBytes(jsonPath), + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip + }); + + if (manifest == null) + { + Log.LogError($"Could not parse manifest from {jsonPath}."); + return false; + } + } + catch (JsonException je) + { + Log.LogError($"Failed to read from {jsonPath}: {je.Message}"); + return false; + } + + if (manifest.DependsOn != null) + { + foreach ((string depName, string depVersion) in manifest.DependsOn) + { + if (!InstallWorkloadManifest(depName, depVersion, nugetConfigContents, stopOnMissing: false)) + { + Log.LogWarning($"Could not install manifest {depName}/{depVersion}. This can be ignored if the workload {WorkloadId.ItemSpec} doesn't depend on it."); + continue; + } + } + } + + return true; + } + + private bool HasMetadata(ITaskItem item, string itemName, string metadataName) + { + if (!string.IsNullOrEmpty(item.GetMetadata(metadataName))) + return true; + + Log.LogError($"{itemName} item ({item.ItemSpec}) is missing Name metadata"); + return false; + } + + private string FindSubDirIgnoringCase(string parentDir, string dirName) + { + IEnumerable<string> matchingDirs = Directory.EnumerateDirectories(parentDir, + dirName, + new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive }); + + string? first = matchingDirs.FirstOrDefault(); + if (matchingDirs.Count() > 1) + { + Log.LogWarning($"Found multiple directories with names that differ only in case. {string.Join(", ", matchingDirs.ToArray())}" + + $"{Environment.NewLine}Using the first one: {first}"); + } + + return first ?? Path.Combine(parentDir, dirName); + } + + private record ManifestInformation( + object Version, + string Description, + + [property: JsonPropertyName("depends-on")] + IDictionary<string, string> DependsOn, + IDictionary<string, WorkloadInformation> Workloads, + IDictionary<string, PackVersionInformation> Packs, + object Data + ); + + private record WorkloadInformation( + bool Abstract, + string Kind, + string Description, + + List<string> Packs, + List<string> Extends, + List<string> Platforms + ); + + private record PackVersionInformation( + string Kind, + string Version, + [property: JsonPropertyName("alias-to")] + Dictionary<string, string> AliasTo + ); + } + + internal record PackageReference(string Name, + string Version, + string OutputDir, + string relativeSourceDir = ""); +} diff --git a/src/tasks/WorkloadBuildTasks/PackageInstaller.cs b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs new file mode 100644 index 00000000000..f6d4e85cf76 --- /dev/null +++ b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs @@ -0,0 +1,163 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +#nullable enable + +namespace Microsoft.Workload.Build.Tasks +{ + internal class PackageInstaller + { + private readonly string _tempDir; + private string _nugetConfigContents; + private TaskLoggingHelper _logger; + private string _packagesDir; + + private PackageInstaller(string nugetConfigContents, TaskLoggingHelper logger) + { + _nugetConfigContents = nugetConfigContents; + + _logger = logger; + _tempDir = Path.Combine(Path.GetTempPath(), "install-workload", Path.GetRandomFileName()); + _packagesDir = Path.Combine(_tempDir, "nuget-packages"); + } + + public static bool Install(PackageReference[] references, string nugetConfigContents, TaskLoggingHelper logger, bool stopOnMissing=true) + { + if (!references.Any()) + return true; + + return new PackageInstaller(nugetConfigContents, logger) + .InstallActual(references, stopOnMissing); + } + + private bool InstallActual(PackageReference[] references, bool stopOnMissing) + { + // Restore packages + if (Directory.Exists(_packagesDir)) + { + _logger.LogMessage(MessageImportance.Low, $"Deleting {_packagesDir}"); + Directory.Delete(_packagesDir, recursive: true); + } + + var projecDir = Path.Combine(_tempDir, "restore"); + var projectPath = Path.Combine(projecDir, "Restore.csproj"); + + Directory.CreateDirectory(projecDir); + + File.WriteAllText(Path.Combine(projecDir, "Directory.Build.props"), "<Project />"); + File.WriteAllText(Path.Combine(projecDir, "Directory.Build.targets"), "<Project />"); + File.WriteAllText(projectPath, GenerateProject(references)); + File.WriteAllText(Path.Combine(projecDir, "nuget.config"), _nugetConfigContents); + + _logger.LogMessage(MessageImportance.Low, $"Restoring packages: {string.Join(", ", references.Select(r => $"{r.Name}/{r.Version}"))}"); + + string args = $"restore \"{projectPath}\" /p:RestorePackagesPath=\"{_packagesDir}\""; + (int exitCode, string output) = Utils.TryRunProcess("dotnet", args, silent: false, debugMessageImportance: MessageImportance.Low); + if (exitCode != 0) + { + LogErrorOrWarning($"Restoring packages failed with exit code: {exitCode}. Output:{Environment.NewLine}{output}", stopOnMissing); + return false; + } + + IList<(PackageReference, string)> failedToRestore = references + .Select(r => (r, Path.Combine(_packagesDir, r.Name.ToLower(), r.Version))) + .Where(tuple => !Directory.Exists(tuple.Item2)) + .ToList(); + + if (failedToRestore.Count > 0) + { + _logger.LogMessage(MessageImportance.Normal, output); + foreach ((PackageReference pkgRef, string pkgDir) in failedToRestore) + LogErrorOrWarning($"Could not restore {pkgRef.Name}/{pkgRef.Version} (can't find {pkgDir})", stopOnMissing); + + return false; + } + + return LayoutPackages(references, stopOnMissing); + } + + private bool LayoutPackages(IEnumerable<PackageReference> references, bool stopOnMissing) + { + foreach (var pkgRef in references) + { + var source = Path.Combine(_packagesDir, pkgRef.Name.ToLower(), pkgRef.Version, pkgRef.relativeSourceDir); + if (!Directory.Exists(source)) + { + LogErrorOrWarning($"Failed to restore {pkgRef.Name}/{pkgRef.Version} (could not find {source})", stopOnMissing); + if (stopOnMissing) + return false; + } + else + { + if (!CopyDirectoryAfresh(source, pkgRef.OutputDir) && stopOnMissing) + return false; + } + } + + return true; + } + + private static string GenerateProject(IEnumerable<PackageReference> references) + { + StringBuilder projectFileBuilder = new(); + projectFileBuilder.Append(@" +<Project Sdk=""Microsoft.NET.Sdk""> + <PropertyGroup> + <TargetFramework>net6.0</TargetFramework> + </PropertyGroup> + <ItemGroup>"); + + foreach (var reference in references) + projectFileBuilder.AppendLine($"<PackageReference Include=\"{reference.Name}\" Version=\"{reference.Version}\" />"); + + projectFileBuilder.Append(@" + </ItemGroup> +</Project> +"); + + return projectFileBuilder.ToString(); + } + + private bool CopyDirectoryAfresh(string srcDir, string destDir) + { + try + { + if (Directory.Exists(destDir)) + { + _logger.LogMessage(MessageImportance.Low, $"Deleting {destDir}"); + Directory.Delete(destDir, recursive: true); + } + + _logger.LogMessage(MessageImportance.Low, $"Copying {srcDir} to {destDir}"); + Directory.CreateDirectory(destDir); + Utils.DirectoryCopy(srcDir, destDir); + + return true; + } + catch (Exception ex) + { + _logger.LogError($"Failed while copying {srcDir} => {destDir}: {ex.Message}"); + if (ex is IOException) + return false; + + throw; + } + } + + private void LogErrorOrWarning(string msg, bool stopOnMissing) + { + if (stopOnMissing) + _logger.LogError(msg); + else + _logger.LogWarning(msg); + } + } +} diff --git a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj index 328672b4514..537418ef31d 100644 --- a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj +++ b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj @@ -6,6 +6,8 @@ <NoWarn>$(NoWarn),CA1050</NoWarn> </PropertyGroup> <ItemGroup> + <Compile Include="..\Common\Utils.cs" /> + <PackageReference Include="Microsoft.Build" Version="$(RefOnlyMicrosoftBuildVersion)" /> <PackageReference Include="Microsoft.Build.Framework" Version="$(RefOnlyMicrosoftBuildFrameworkVersion)" /> <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(RefOnlyMicrosoftBuildTasksCoreVersion)" /> |