diff options
author | Larry Ewing <lewing@microsoft.com> | 2022-08-22 08:02:46 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-22 08:02:46 +0300 |
commit | 754014e711fe38dcf8062a12a1c248227b5b0526 (patch) | |
tree | b6cef9a88885d2626fcb6f07a0acaa0ec73e9dda /src/tasks | |
parent | 37ddbefcaf0e768218c2d2e782ef810f8bb1dd8b (diff) |
[release/7.0-rc1] net7.0 + net6.0 workload multitargeting changes (#74018)
* WIP
* Update all the workload names to be net6 versions. This is needed in 7 but NOT in 6.
Dotnet workload install wasm-tools-net6 should install a different set of components than wasm-tools
* Add localization to net7 and correct localization for net6 to target that workload
* Update to 6.0.9 which should be the next version of emsdk.
* Update the net7 and net6 workloads to rename all sdk packs
* Use localize data from net7 project
* Fix typo
* Fix typo
* Update KnownRuntimePack mono versions without clearing them all
* Add net6.0 to description and fix up emscripten-net6 reference
* Fix emsdk version number
* depend on microsoft-net-sdk-emscripten-net7
* Fix missing .net7 renames
* Update WorkloadManifest.targets.in
Don't use **FromWorkload** for pack versions because it doesn't understand the name aliasing we have to use with runtime packs
* Update workload manifest to fix names
* Update the emscripten manifest dependencies
* Use full hash
* WIP: install workload combos for testing
* Update Versions.props
Use Pacjjag6.0.8 packages for the moment because they should be in dotnet-public
* Update emsdk dependencies to `7.0.0-rc.1.22418.3`
* Update emscripten to 7.0.0-rc.1.22418.6, and 6.0.4
* wip
* Fix conditions in WorkloadManifest.targets
* Fix up running WBT
* Fix build
HACK: to work around an incorrect condition in emscripten's
WorkloadManifest.targets which causes emscripten packs not to get
imported.
* Update WorkloadManifest.targets.in
* Update WorkloadManifest.targets.in
* Fix dotnet path references for workload testing
* InstallWorkloadFromArtifacts: don't delete the target path between different requests, clean up before starting
* Fix PackageVersionNet6=6.0.8
* WorkloadManifest.targets: rename some properties to be internal, and versioned
* More renames to fix builds
* Update Versions.props
* Fix up a condition in WorkloadManifest.targets
* Work around the import troubles
* Work around the import troubles
* Revert broken version check
* Use BrowserWorkloadDisabled which will be set in both cases
* Fix typos in WorkloadManifest.targets
* Wasm.Build.Tests: Workaround a msbuild bug causing mysterious failures
When running a fresh build with `dotnet build`, the first one works
fine. But all the subsequent builds fail with no info in the logs at
all.
```
$ /workspaces/runtime/artifacts/bin/dotnet-net7/dotnet build
MSBuild version 17.4.0-preview-22413-01+f0a66ec43 for .NET
Build FAILED.
0 Warning(s)
0 Error(s)
```
And it seems to be related to msbuild nodes, so disable NodeReuse.
* cleanup
* Remove a bunch of stale logic from WBT
Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
Co-authored-by: Marc Paine <marcpop@microsoft.com>
Co-authored-by: Ankit Jain <radical@gmail.com>
Diffstat (limited to 'src/tasks')
-rw-r--r-- | src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 6 | ||||
-rw-r--r-- | src/tasks/Common/Utils.cs | 7 | ||||
-rw-r--r-- | src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs | 244 |
3 files changed, 204 insertions, 53 deletions
diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 291af85bde4..55e93530680 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -488,15 +488,11 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task { // compare original assembly vs it's outputs.. all it's outputs! string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); - if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) + if (args.ProxyFiles.Any(pf => Utils.IsNewerThan(assemblyPath, pf.TargetFile))) return false; } return true; - - static bool IsNewerThanOutput(string inFile, string outFile) - => !File.Exists(inFile) || !File.Exists(outFile) || - (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); } private IEnumerable<ITaskItem> FilterOutUnmanagedAssemblies(IEnumerable<ITaskItem> assemblies) diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 2c624214937..8ed6c2c1371 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -23,6 +23,10 @@ internal static class Utils return reader.ReadToEnd(); } + public static bool IsNewerThan(string inFile, string outFile) + => !File.Exists(inFile) || !File.Exists(outFile) || + (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); + public static (int exitCode, string output) RunShellCommand( TaskLoggingHelper logger, string command, @@ -233,6 +237,9 @@ internal static class Utils #if NETCOREAPP public static void DirectoryCopy(string sourceDir, string destDir, Func<string, bool>? predicate=null) { + if (!Directory.Exists(destDir)) + Directory.CreateDirectory(destDir); + string[] files = Directory.GetFiles(sourceDir, "*", SearchOption.AllDirectories); foreach (string file in files) { diff --git a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs index 2fd3597425e..1e77d5905f5 100644 --- a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs +++ b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs @@ -20,7 +20,10 @@ namespace Microsoft.Workload.Build.Tasks public class InstallWorkloadFromArtifacts : Task { [Required, NotNull] - public ITaskItem[] WorkloadIds { get; set; } = Array.Empty<ITaskItem>(); + public ITaskItem[] WorkloadIds { get; set; } = Array.Empty<ITaskItem>(); + + [Required, NotNull] + public ITaskItem[] InstallTargets { get; set; } = Array.Empty<ITaskItem>(); [Required, NotNull] public string? VersionBand { get; set; } @@ -32,22 +35,71 @@ namespace Microsoft.Workload.Build.Tasks public string? TemplateNuGetConfigPath { get; set; } [Required, NotNull] - public string? SdkDir { get; set; } + public string SdkWithNoWorkloadInstalledPath { get; set; } = string.Empty; public bool OnlyUpdateManifests{ get; set; } private const string s_nugetInsertionTag = "<!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->"; + private string AllManifestsStampPath => Path.Combine(SdkWithNoWorkloadInstalledPath, ".all-manifests.stamp"); public override bool Execute() { try { - foreach (var workloadIdItem in WorkloadIds) + if (!Directory.Exists(SdkWithNoWorkloadInstalledPath)) + throw new LogAsErrorException($"Cannot find {nameof(SdkWithNoWorkloadInstalledPath)}={SdkWithNoWorkloadInstalledPath}"); + + if (!Directory.Exists(LocalNuGetsPath)) + throw new LogAsErrorException($"Cannot find {nameof(LocalNuGetsPath)}={LocalNuGetsPath} . " + + "Set it to the Shipping packages directory in artifacts."); + + if (!InstallAllManifests()) + return false; + + if (OnlyUpdateManifests) + return !Log.HasLoggedErrors; + + InstallWorkloadRequest[] selectedRequests = InstallTargets + .SelectMany(workloadToInstall => + { + if (!HasMetadata(workloadToInstall, nameof(workloadToInstall), "Variants", Log)) + throw new LogAsErrorException($"Missing Variants metadata on item '{workloadToInstall.ItemSpec}'"); + + return workloadToInstall + .GetMetadata("Variants") + .Split(";", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(v => (variant: v, target: workloadToInstall)); + }) + .SelectMany(w => + { + IEnumerable<InstallWorkloadRequest> workloads = WorkloadIds.Where(wi => wi.GetMetadata("Variant") == w.variant) + .Select(wi => new InstallWorkloadRequest(wi, w.target)); + return workloads.Any() + ? workloads + : throw new LogAsErrorException($"Could not find any workload variant named '{w.variant}'"); + }).ToArray(); + + foreach (InstallWorkloadRequest req in selectedRequests) { - if (!ExecuteInternal(workloadIdItem)) + if (Directory.Exists(req.TargetPath)) + { + Log.LogMessage(MessageImportance.Low, $"Deleting directory {req.TargetPath}"); + Directory.Delete(req.TargetPath, recursive: true); + } + } + + string cachePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + foreach (InstallWorkloadRequest req in selectedRequests) + { + Log.LogMessage(MessageImportance.High, $"** Installing workload {req.WorkloadId} in {req.TargetPath} **"); + if (!req.Validate(Log)) + return false; + + if (!ExecuteInternal(req) && !req.IgnoreErrors) return false; } - return true; + + return !Log.HasLoggedErrors; } catch (LogAsErrorException laee) { @@ -56,73 +108,136 @@ namespace Microsoft.Workload.Build.Tasks } } - private bool ExecuteInternal(ITaskItem workloadId) + private bool ExecuteInternal(InstallWorkloadRequest req) { - if (!HasMetadata(workloadId, nameof(workloadId), "Version") || - !HasMetadata(workloadId, nameof(workloadId), "ManifestName")) + if (!File.Exists(TemplateNuGetConfigPath)) { + Log.LogError($"Cannot find TemplateNuGetConfigPath={TemplateNuGetConfigPath}"); return false; } - if (!Directory.Exists(SdkDir)) - { - Log.LogError($"Cannot find SdkDir={SdkDir}"); + Log.LogMessage(MessageImportance.Low, $"Duplicating {SdkWithNoWorkloadInstalledPath} into {req.TargetPath}"); + Utils.DirectoryCopy(SdkWithNoWorkloadInstalledPath, req.TargetPath); + + string nugetConfigContents = GetNuGetConfig(); + if (!InstallPacks(req, nugetConfigContents)) return false; + + UpdateAppRef(req.TargetPath, req.Version); + + return !Log.HasLoggedErrors; + } + + private bool InstallAllManifests() + { + var allManifestPkgs = Directory.EnumerateFiles(LocalNuGetsPath, "*Manifest*nupkg"); + if (!AnyInputsNewerThanOutput(AllManifestsStampPath, allManifestPkgs)) + { + Log.LogMessage(MessageImportance.Low, + $"Skipping installing manifests because the {AllManifestsStampPath} " + + $"is newer than packages {string.Join(',', allManifestPkgs)}."); + return true; } - if (!File.Exists(TemplateNuGetConfigPath)) + // HACK BEGIN - because sdk doesn't yet have the net6/net7 manifest names in the known workloads + // list + string? txtPath = Directory.EnumerateFiles(Path.Combine(SdkWithNoWorkloadInstalledPath, "sdk"), "IncludedWorkloadManifests.txt", + new EnumerationOptions { RecurseSubdirectories = true, MaxRecursionDepth = 2}) + .FirstOrDefault(); + if (txtPath is null) + throw new LogAsErrorException($"Could not find IncludedWorkloadManifests.txt in {SdkWithNoWorkloadInstalledPath}"); + + string stampPath = Path.Combine(Path.GetDirectoryName(txtPath)!, ".stamp"); + if (!File.Exists(stampPath)) { - Log.LogError($"Cannot find TemplateNuGetConfigPath={TemplateNuGetConfigPath}"); - return false; + Log.LogMessage(MessageImportance.High, $"txtPath: {txtPath}"); + string newTxt = File.ReadAllText(txtPath) + .Replace("microsoft.net.workload.mono.toolchain", + $"microsoft.net.workload.mono.toolchain.net6{Environment.NewLine}microsoft.net.workload.mono.toolchain.net7") + .Replace("microsoft.net.workload.emscripten", + $"microsoft.net.workload.emscripten.net6{Environment.NewLine}microsoft.net.workload.emscripten.net7"); + File.WriteAllText(txtPath, newTxt); + File.WriteAllText(stampPath, ""); } - Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Installing workload manifest {workloadId.ItemSpec} **{Environment.NewLine}"); + string p = Path.Combine(SdkWithNoWorkloadInstalledPath, "sdk-manifests", "7.0.100", "microsoft.net.workload.mono.toolchain"); + Log.LogMessage(MessageImportance.High, $"Deleting {p}"); + if (Directory.Exists(p)) + Directory.Delete(p, recursive: true); + p = Path.Combine(SdkWithNoWorkloadInstalledPath, "sdk-manifests", "7.0.100", "microsoft.net.workload.emscripten"); + Log.LogMessage(MessageImportance.High, $"Deleting {p}"); + if (Directory.Exists(p)) + Directory.Delete(p, recursive: true); + // HACK END string nugetConfigContents = GetNuGetConfig(); - if (!InstallWorkloadManifest(workloadId, workloadId.GetMetadata("ManifestName"), workloadId.GetMetadata("Version"), nugetConfigContents, stopOnMissing: true)) - return false; + HashSet<string> manifestsInstalled = new(); + foreach (ITaskItem workload in WorkloadIds) + { + InstallWorkloadRequest req = new(workload, new TaskItem()); - if (OnlyUpdateManifests) - return !Log.HasLoggedErrors; + if (manifestsInstalled.Contains(req.ManifestName)) + { + Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Manifests for workload {req.WorkloadId} are already installed **{Environment.NewLine}"); + continue; + } - if (!InstallPacks(workloadId, nugetConfigContents)) - return false; + Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Installing manifests for workload {req.WorkloadId} **"); + if (!InstallWorkloadManifest(workload, + req.ManifestName, + req.Version, + SdkWithNoWorkloadInstalledPath, + nugetConfigContents, + stopOnMissing: true)) + { + return false; + } - UpdateAppRef(workloadId.GetMetadata("Version")); + manifestsInstalled.Add(req.ManifestName); + } - return !Log.HasLoggedErrors; + File.WriteAllText(AllManifestsStampPath, string.Empty); + + return true; } - private bool InstallPacks(ITaskItem workloadId, string nugetConfigContents) + private bool InstallPacks(InstallWorkloadRequest req, string nugetConfigContents) { string nugetConfigPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); File.WriteAllText(nugetConfigPath, nugetConfigContents); - Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** workload install **{Environment.NewLine}"); + // Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** dotnet workload install {req.WorkloadId} **{Environment.NewLine}"); (int exitCode, string output) = Utils.TryRunProcess( Log, - Path.Combine(SdkDir, "dotnet"), - $"workload install --skip-manifest-update --no-cache --configfile \"{nugetConfigPath}\" {workloadId.ItemSpec}", + Path.Combine(req.TargetPath, "dotnet"), + $"workload install --skip-manifest-update --no-cache --configfile \"{nugetConfigPath}\" {req.WorkloadId}", workingDir: Path.GetTempPath(), silent: false, + logStdErrAsMessage: req.IgnoreErrors, debugMessageImportance: MessageImportance.High); if (exitCode != 0) { - Log.LogError($"workload install failed with exit code {exitCode}: {output}"); - - foreach (string dir in Directory.EnumerateDirectories(Path.Combine(SdkDir, "sdk-manifests"), "*", SearchOption.AllDirectories)) - Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(SdkDir, "sdk-manifests", dir)}"); + if (req.IgnoreErrors) + { + Log.LogMessage(MessageImportance.High, + $"{Environment.NewLine} ** Ignoring workload installation failure exit code {exitCode}. **{Environment.NewLine}"); + } + else + { + Log.LogError($"workload install failed with exit code {exitCode}: {output}"); + } - foreach (string dir in Directory.EnumerateDirectories(Path.Combine(SdkDir, "packs"), "*", SearchOption.AllDirectories)) - Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(SdkDir, "packs", dir)}"); + foreach (string dir in Directory.EnumerateDirectories(Path.Combine(req.TargetPath, "sdk-manifests"), "*", SearchOption.AllDirectories)) + Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(req.TargetPath, "sdk-manifests", dir)}"); - return false; + foreach (string dir in Directory.EnumerateDirectories(Path.Combine(req.TargetPath, "packs"), "*", SearchOption.AllDirectories)) + Log.LogMessage(MessageImportance.Low, $"\t{Path.Combine(req.TargetPath, "packs", dir)}"); } return !Log.HasLoggedErrors; } - private void UpdateAppRef(string version) + private void UpdateAppRef(string sdkPath, string version) { Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** Updating Targeting pack **{Environment.NewLine}"); @@ -131,7 +246,7 @@ namespace Microsoft.Workload.Build.Tasks throw new LogAsErrorException($"Could not find {pkgPath} needed to update the targeting pack to the newly built one." + " Make sure to build the subset `packs`, like `./build.sh -os browser -s mono+libs+packs`."); - string packDir = Path.Combine(SdkDir, "packs", "Microsoft.NETCore.App.Ref"); + string packDir = Path.Combine(sdkPath, "packs", "Microsoft.NETCore.App.Ref"); string[] dirs = Directory.EnumerateDirectories(packDir).ToArray(); if (dirs.Length != 1) throw new LogAsErrorException($"Expected to find exactly one versioned directory under {packDir}, but got " + @@ -150,28 +265,28 @@ namespace Microsoft.Workload.Build.Tasks private string GetNuGetConfig() { string contents = File.ReadAllText(TemplateNuGetConfigPath); - if (contents.IndexOf(s_nugetInsertionTag) < 0) + if (contents.IndexOf(s_nugetInsertionTag, StringComparison.InvariantCultureIgnoreCase) < 0) throw new LogAsErrorException($"Could not find {s_nugetInsertionTag} in {TemplateNuGetConfigPath}"); return contents.Replace(s_nugetInsertionTag, $@"<add key=""nuget-local"" value=""{LocalNuGetsPath}"" />"); } - private bool InstallWorkloadManifest(ITaskItem workloadId, string name, string version, string nugetConfigContents, bool stopOnMissing) + private bool InstallWorkloadManifest(ITaskItem workloadId, string name, string version, string sdkDir, string nugetConfigContents, bool stopOnMissing) { - Log.LogMessage(MessageImportance.High, $"Installing workload manifest for {name}/{version}"); + Log.LogMessage(MessageImportance.High, $" ** Installing manifest: {name}/{version}"); // Find any existing directory with the manifest name, ignoring the case // Multiple directories for a manifest, differing only in case causes // workload install to fail due to duplicate manifests! // This is applicable only on case-sensitive filesystems - string outputDir = FindSubDirIgnoringCase(Path.Combine(SdkDir, "sdk-manifests", VersionBand), name); + string outputDir = FindSubDirIgnoringCase(Path.Combine(sdkDir, "sdk-manifests", VersionBand), name); PackageReference pkgRef = new(Name: $"{name}.Manifest-{VersionBand}", Version: version, OutputDir: outputDir, relativeSourceDir: "data"); - if (!PackageInstaller.Install(new[]{ pkgRef }, nugetConfigContents, Log, stopOnMissing)) + if (!PackageInstaller.Install(new[] { pkgRef }, nugetConfigContents, Log, stopOnMissing)) return false; string manifestDir = pkgRef.OutputDir; @@ -209,7 +324,7 @@ namespace Microsoft.Workload.Build.Tasks { foreach ((string depName, string depVersion) in manifest.DependsOn) { - if (!InstallWorkloadManifest(workloadId, depName, depVersion, nugetConfigContents, stopOnMissing: false)) + if (!InstallWorkloadManifest(workloadId, depName, depVersion, sdkDir, nugetConfigContents, stopOnMissing: false)) { Log.LogWarning($"Could not install manifest {depName}/{depVersion}. This can be ignored if the workload {workloadId.ItemSpec} doesn't depend on it."); continue; @@ -220,31 +335,35 @@ namespace Microsoft.Workload.Build.Tasks return true; } - private bool HasMetadata(ITaskItem item, string itemName, string metadataName) + private static bool HasMetadata(ITaskItem item, string itemName, string metadataName, TaskLoggingHelper log) { if (!string.IsNullOrEmpty(item.GetMetadata(metadataName))) return true; - Log.LogError($"{itemName} item ({item.ItemSpec}) is missing Name metadata"); + log.LogError($"{itemName} item ({item.ItemSpec}) is missing {metadataName} metadata"); return false; } private string FindSubDirIgnoringCase(string parentDir, string dirName) { - IEnumerable<string> matchingDirs = Directory.EnumerateDirectories(parentDir, + string[] matchingDirs = Directory.EnumerateDirectories(parentDir, dirName, - new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive }); + new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive }) + .ToArray(); string? first = matchingDirs.FirstOrDefault(); - if (matchingDirs.Count() > 1) + if (matchingDirs.Length > 1) { - Log.LogWarning($"Found multiple directories with names that differ only in case. {string.Join(", ", matchingDirs.ToArray())}" + Log.LogWarning($"Found multiple directories with names that differ only in case. {string.Join(", ", matchingDirs)}" + $"{Environment.NewLine}Using the first one: {first}"); } return first ?? Path.Combine(parentDir, dirName.ToLower(CultureInfo.InvariantCulture)); } + private static bool AnyInputsNewerThanOutput(string output, IEnumerable<string> inputs) + => inputs.Any(i => Utils.IsNewerThan(i, output)); + private sealed record ManifestInformation( object Version, string Description, @@ -272,6 +391,35 @@ namespace Microsoft.Workload.Build.Tasks [property: JsonPropertyName("alias-to")] Dictionary<string, string> AliasTo ); + + internal sealed record InstallWorkloadRequest( + ITaskItem Workload, + ITaskItem Target) + { + public string ManifestName => Workload.GetMetadata("ManifestName"); + public string Version => Workload.GetMetadata("Version"); + public string TargetPath => Target.GetMetadata("InstallPath"); + public bool IgnoreErrors => Workload.GetMetadata("IgnoreErrors").ToLowerInvariant() == "true"; + public string WorkloadId => Workload.ItemSpec; + + public bool Validate(TaskLoggingHelper log) + { + if (!HasMetadata(Workload, nameof(Workload), "Version", log) || + !HasMetadata(Workload, nameof(Workload), "ManifestName", log) || + !HasMetadata(Target, nameof(Target), "InstallPath", log)) + { + return false; + } + + if (string.IsNullOrEmpty(TargetPath)) + { + log.LogError($"InstallPath is empty for workload {Workload.ItemSpec}"); + return false; + } + + return true; + } + } } internal sealed record PackageReference(string Name, |