[netcore] Replace dl-test-assets.py with msbuild download (#15883)

The python file was unreliable on macOS, let's see whether a simple msbuild replacement is better.

Also fixed dependencies on some of the Makefile targets so that they correctly detect when they need to run.

4 files changed, 56 insertions, 54 deletions

diff --git a/netcore/.gitignore b/netcore/.gitignore
index 865a7d36605..2e52576a08b 100644
--- a/netcore/.gitignore
+++ b/netcore/.gitignore
@@ -11,7 +11,6 @@ packs/
 /corefx
 /roslyn
 /obj
-corefx-test-assets.xml
 .configured
 /System.Private.CoreLib/src/System/Environment.Mono.cs
 /config.make
diff --git a/netcore/Makefile b/netcore/Makefile
index 903fe790236..1916d958816 100644
--- a/netcore/Makefile
+++ b/netcore/Makefile
@@ -50,7 +50,7 @@ ASPNETCORESDK_FILE := aspnetcore-runtime-$(ASPNETCOREAPP_VERSION)-$(RID).$(NETCO
 NETCORE_URL := https://dotnetcli.blob.core.windows.net/dotnet/Runtime/$(NETCOREAPP_VERSION)/$(NETCORESDK_FILE)
 ASPNETCORE_URL := https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/$(ASPNETCOREAPP_VERSION)/$(ASPNETCORESDK_FILE)
 FEED_BASE_URL := https://dotnetfeed.blob.core.windows.net/dotnet-core
-TEST_ASSETS_URL = $(FEED_BASE_URL)/corefx-tests/$(NETCORETESTS_VERSION)/$(TESTS_PLATFORM)/netcoreapp/corefx-test-assets.xml
+TEST_ASSETS_PATH := corefx-tests/$(NETCORETESTS_VERSION)/$(TESTS_PLATFORM)/netcoreapp/corefx-test-assets.xml
 
 # used to calculate exact version number for generating nupkg
 BLAME = $(shell git blame ../configure.ac | grep AC_INIT | cut -f1 -d' ')
@@ -77,7 +77,9 @@ corefx/.stamp-dl-corefx-$(NETCORETESTS_VERSION): corefx-restore.csproj
 	$(DOTNET) build corefx-restore.csproj --runtime $(RID) --packages corefx/packages -p:MicrosoftPrivateCoreFxNETCoreAppVersion=$(NETCORETESTS_VERSION) -p:OutputPath=corefx/restore/
 	touch $@
 
-update-roslyn: roslyn-restore.csproj
+update-roslyn: roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec
+
+roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec:
 	$(DOTNET) restore roslyn-restore.csproj -p:RoslynVersion=$(ROSLYN_VERSION) --packages roslyn/packages -p:OutputPath=roslyn/restore/
 
 run-sample: prepare
@@ -110,10 +112,13 @@ debug-bcl:
 runtime:
 	$(MAKE) -C ../mono
 
-link-mono:
+link-mono: $(SHAREDRUNTIME)/.stamp-link-mono
+
+$(SHAREDRUNTIME)/.stamp-link-mono: ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb
 	cp ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) $(SHAREDRUNTIME)/$(PLATFORM_AOT_PREFIX)coreclr$(PLATFORM_AOT_SUFFIX)
 	cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll $(SHAREDRUNTIME)
 	cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb $(SHAREDRUNTIME)
+	touch $@
 
 prepare: $(ASPNETCORESDK_FILE) $(NETCORESDK_FILE) update-corefx update-roslyn link-mono
 
@@ -128,8 +133,8 @@ clean:
 update-tests-corefx: corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION)
 
 corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION):
-	curl $(TEST_ASSETS_URL) --output corefx-test-assets.xml
-	./dl-test-assets.py corefx-test-assets.xml $(FEED_BASE_URL) corefx/tests
+	rm -rf corefx/tests
+	$(DOTNET) msbuild corefx-tests-restore.proj -m -t:DownloadAllTests -p:TEST_ASSETS_PATH="$(TEST_ASSETS_PATH)" -p:FEED_BASE_URL="$(FEED_BASE_URL)"
 	rm -rf corefx/tests/extracted/System.Utf8String.Experimental.Tests
 	touch $@
 
@@ -151,12 +156,13 @@ run-tests-corefx: update-tests-corefx
 	$(MAKE) xunit-summary
 
 run-tests-corefx-%: prepare update-tests-corefx
-	echo -n "***************** $* *********************"
-	cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/xunit.console.deps.json
-	cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/Microsoft.DotNet.RemoteExecutorHost.deps.json
-	cp corefx/restore/corefx-restore.runtimeconfig.dev.json corefx/tests/extracted/$*/$*.runtimeconfig.dev.json
-	cp corefx/restore/corefx-restore.dll corefx/tests/extracted/$*/corefx-restore.dll
-	sed -i -e 's/5.0.0\"/$(NETCOREAPP_VERSION)\"/g' corefx/tests/extracted/$*/*.runtimeconfig.json
+	@echo ""
+	@echo "***************** $* *********************"
+	@cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/xunit.console.deps.json
+	@cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/Microsoft.DotNet.RemoteExecutorHost.deps.json
+	@cp corefx/restore/corefx-restore.runtimeconfig.dev.json corefx/tests/extracted/$*/$*.runtimeconfig.dev.json
+	@cp corefx/restore/corefx-restore.dll corefx/tests/extracted/$*/corefx-restore.dll
+	@sed -i -e 's/5.0.0\"/$(NETCOREAPP_VERSION)\"/g' corefx/tests/extracted/$*/*.runtimeconfig.json
 	cd corefx/tests/extracted/$* && \
 	COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet exec --runtimeconfig $*.runtimeconfig.json --fx-version "$(NETCOREAPP_VERSION)" xunit.console.dll $*.dll \
 		-html ../../TestResult-$*.html -xml ../../TestResult-$*-netcore-xunit.xml \
diff --git a/netcore/corefx-tests-restore.proj b/netcore/corefx-tests-restore.proj
new file mode 100644
index 00000000000..1cd7b8f207a
--- /dev/null
+++ b/netcore/corefx-tests-restore.proj
@@ -0,0 +1,39 @@
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+    <Target Name="DownloadTest">
+        <DownloadFile
+            SourceUrl="$(FEED_BASE_URL)/$(TestFile)"
+            DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests"
+            Retries="3">
+            <Output TaskParameter="DownloadedFile" ItemName="Content" />
+        </DownloadFile>
+    </Target>
+
+    <Target Name="DownloadTestsList">
+        <DownloadFile
+            SourceUrl="$(FEED_BASE_URL)/$(TEST_ASSETS_PATH)"
+            DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests"
+            Retries="3">
+            <Output TaskParameter="DownloadedFile" ItemName="Content" />
+        </DownloadFile>
+        <XmlPeek
+            XmlInputPath="$(MSBuildProjectDirectory)/corefx/tests/corefx-test-assets.xml" 
+            Query="/Build/Blob/@Id">
+            <Output TaskParameter="Result" ItemName="AllTests" />
+        </XmlPeek>
+    </Target>
+
+    <Target Name="DownloadAllTests" DependsOnTargets="DownloadTestsList">
+        <ItemGroup>
+            <TempProjects Include="$(MSBuildProjectFile)">
+                <Properties>TestFile=%(AllTests.Identity)</Properties>
+            </TempProjects>
+        </ItemGroup>
+        <MSBuild Projects="@(TempProjects)" BuildInParallel="true" StopOnFirstFailure="true" Targets="DownloadTest" />
+        <Message Text="Extracting CoreFX tests..." Importance="high" />
+        <Unzip
+            SourceFiles="$(MSBuildProjectDirectory)/corefx/tests/$([System.IO.Path]::GetFileName('%(AllTests.Identity)'))"
+            DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests/extracted/$([System.IO.Path]::GetFileNameWithoutExtension('%(AllTests.Identity)'))" />
+    </Target>
+
+</Project>
diff --git a/netcore/dl-test-assets.py b/netcore/dl-test-assets.py
deleted file mode 100755
index d422801b513..00000000000
--- a/netcore/dl-test-assets.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import os
-import subprocess
-import xml.etree.ElementTree as ET
-import zipfile
-import urllib.request, urllib.parse, urllib.error
-import multiprocessing
-
-if len(sys.argv) < 4:
-    print("Usage: dl-test-assets.py <path to assets.xml> <base url> <output dir>")
-    sys.exit(1)
-
-infile_name = sys.argv [1]
-base_url = sys.argv [2]
-outdir = sys.argv [3]
-tree = ET.parse(infile_name)
-root = tree.getroot()
-
-if not os.path.exists(outdir):
-    os.makedirs(outdir)
-
-def downloadAsset(elem):
-    if elem.tag != "Blob":
-        return
-    id = elem.attrib ["Id"]
-    filename = os.path.basename(id)
-    print("Downloading " + filename)
-    try:
-        name, hdrs = urllib.request.urlretrieve(base_url + "/" + id, outdir + "/" + filename)
-    except IOError as e:
-        print("Download failed for " + id)
-        sys.exit (1)
-    print("Extracting " + filename)
-    with zipfile.ZipFile(outdir + "/" + filename) as zf:
-        zf.extractall(outdir + "/extracted/" + filename[:-4])
-
-pool = multiprocessing.Pool(multiprocessing.cpu_count())
-results = pool.map(downloadAsset, root)
-pool.close()
-pool.join()