diff options
author | Alexander Köplinger <alex.koeplinger@outlook.com> | 2019-07-30 20:27:42 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-07-30 20:27:42 +0300 |
commit | 9f0d493eea7da2b9494f1e8185740a99fbee00a1 (patch) | |
tree | c24c3441cc38049cfef17082ce91e79c0ec5ea58 /netcore | |
parent | a8d8fb6ceadde1fa0f7ae5dfe4a82f91442877f9 (diff) |
[netcore] Replace dl-test-assets.py with msbuild download (#15883)
The python file was unreliable on macOS, let's see whether a simple msbuild replacement is better.
Also fixed dependencies on some of the Makefile targets so that they correctly detect when they need to run.
Diffstat (limited to 'netcore')
-rw-r--r-- | netcore/.gitignore | 1 | ||||
-rw-r--r-- | netcore/Makefile | 28 | ||||
-rw-r--r-- | netcore/corefx-tests-restore.proj | 39 | ||||
-rwxr-xr-x | netcore/dl-test-assets.py | 42 |
4 files changed, 56 insertions, 54 deletions
diff --git a/netcore/.gitignore b/netcore/.gitignore index 865a7d36605..2e52576a08b 100644 --- a/netcore/.gitignore +++ b/netcore/.gitignore @@ -11,7 +11,6 @@ packs/ /corefx /roslyn /obj -corefx-test-assets.xml .configured /System.Private.CoreLib/src/System/Environment.Mono.cs /config.make diff --git a/netcore/Makefile b/netcore/Makefile index 903fe790236..1916d958816 100644 --- a/netcore/Makefile +++ b/netcore/Makefile @@ -50,7 +50,7 @@ ASPNETCORESDK_FILE := aspnetcore-runtime-$(ASPNETCOREAPP_VERSION)-$(RID).$(NETCO NETCORE_URL := https://dotnetcli.blob.core.windows.net/dotnet/Runtime/$(NETCOREAPP_VERSION)/$(NETCORESDK_FILE) ASPNETCORE_URL := https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/$(ASPNETCOREAPP_VERSION)/$(ASPNETCORESDK_FILE) FEED_BASE_URL := https://dotnetfeed.blob.core.windows.net/dotnet-core -TEST_ASSETS_URL = $(FEED_BASE_URL)/corefx-tests/$(NETCORETESTS_VERSION)/$(TESTS_PLATFORM)/netcoreapp/corefx-test-assets.xml +TEST_ASSETS_PATH := corefx-tests/$(NETCORETESTS_VERSION)/$(TESTS_PLATFORM)/netcoreapp/corefx-test-assets.xml # used to calculate exact version number for generating nupkg BLAME = $(shell git blame ../configure.ac | grep AC_INIT | cut -f1 -d' ') @@ -77,7 +77,9 @@ corefx/.stamp-dl-corefx-$(NETCORETESTS_VERSION): corefx-restore.csproj $(DOTNET) build corefx-restore.csproj --runtime $(RID) --packages corefx/packages -p:MicrosoftPrivateCoreFxNETCoreAppVersion=$(NETCORETESTS_VERSION) -p:OutputPath=corefx/restore/ touch $@ -update-roslyn: roslyn-restore.csproj +update-roslyn: roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec + +roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec: $(DOTNET) restore roslyn-restore.csproj -p:RoslynVersion=$(ROSLYN_VERSION) --packages roslyn/packages -p:OutputPath=roslyn/restore/ run-sample: prepare @@ -110,10 +112,13 @@ debug-bcl: runtime: $(MAKE) -C ../mono -link-mono: +link-mono: $(SHAREDRUNTIME)/.stamp-link-mono + +$(SHAREDRUNTIME)/.stamp-link-mono: ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb cp ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) $(SHAREDRUNTIME)/$(PLATFORM_AOT_PREFIX)coreclr$(PLATFORM_AOT_SUFFIX) cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll $(SHAREDRUNTIME) cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb $(SHAREDRUNTIME) + touch $@ prepare: $(ASPNETCORESDK_FILE) $(NETCORESDK_FILE) update-corefx update-roslyn link-mono @@ -128,8 +133,8 @@ clean: update-tests-corefx: corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION) corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION): - curl $(TEST_ASSETS_URL) --output corefx-test-assets.xml - ./dl-test-assets.py corefx-test-assets.xml $(FEED_BASE_URL) corefx/tests + rm -rf corefx/tests + $(DOTNET) msbuild corefx-tests-restore.proj -m -t:DownloadAllTests -p:TEST_ASSETS_PATH="$(TEST_ASSETS_PATH)" -p:FEED_BASE_URL="$(FEED_BASE_URL)" rm -rf corefx/tests/extracted/System.Utf8String.Experimental.Tests touch $@ @@ -151,12 +156,13 @@ run-tests-corefx: update-tests-corefx $(MAKE) xunit-summary run-tests-corefx-%: prepare update-tests-corefx - echo -n "***************** $* *********************" - cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/xunit.console.deps.json - cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/Microsoft.DotNet.RemoteExecutorHost.deps.json - cp corefx/restore/corefx-restore.runtimeconfig.dev.json corefx/tests/extracted/$*/$*.runtimeconfig.dev.json - cp corefx/restore/corefx-restore.dll corefx/tests/extracted/$*/corefx-restore.dll - sed -i -e 's/5.0.0\"/$(NETCOREAPP_VERSION)\"/g' corefx/tests/extracted/$*/*.runtimeconfig.json + @echo "" + @echo "***************** $* *********************" + @cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/xunit.console.deps.json + @cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/Microsoft.DotNet.RemoteExecutorHost.deps.json + @cp corefx/restore/corefx-restore.runtimeconfig.dev.json corefx/tests/extracted/$*/$*.runtimeconfig.dev.json + @cp corefx/restore/corefx-restore.dll corefx/tests/extracted/$*/corefx-restore.dll + @sed -i -e 's/5.0.0\"/$(NETCOREAPP_VERSION)\"/g' corefx/tests/extracted/$*/*.runtimeconfig.json cd corefx/tests/extracted/$* && \ COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet exec --runtimeconfig $*.runtimeconfig.json --fx-version "$(NETCOREAPP_VERSION)" xunit.console.dll $*.dll \ -html ../../TestResult-$*.html -xml ../../TestResult-$*-netcore-xunit.xml \ diff --git a/netcore/corefx-tests-restore.proj b/netcore/corefx-tests-restore.proj new file mode 100644 index 00000000000..1cd7b8f207a --- /dev/null +++ b/netcore/corefx-tests-restore.proj @@ -0,0 +1,39 @@ +<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + + <Target Name="DownloadTest"> + <DownloadFile + SourceUrl="$(FEED_BASE_URL)/$(TestFile)" + DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests" + Retries="3"> + <Output TaskParameter="DownloadedFile" ItemName="Content" /> + </DownloadFile> + </Target> + + <Target Name="DownloadTestsList"> + <DownloadFile + SourceUrl="$(FEED_BASE_URL)/$(TEST_ASSETS_PATH)" + DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests" + Retries="3"> + <Output TaskParameter="DownloadedFile" ItemName="Content" /> + </DownloadFile> + <XmlPeek + XmlInputPath="$(MSBuildProjectDirectory)/corefx/tests/corefx-test-assets.xml" + Query="/Build/Blob/@Id"> + <Output TaskParameter="Result" ItemName="AllTests" /> + </XmlPeek> + </Target> + + <Target Name="DownloadAllTests" DependsOnTargets="DownloadTestsList"> + <ItemGroup> + <TempProjects Include="$(MSBuildProjectFile)"> + <Properties>TestFile=%(AllTests.Identity)</Properties> + </TempProjects> + </ItemGroup> + <MSBuild Projects="@(TempProjects)" BuildInParallel="true" StopOnFirstFailure="true" Targets="DownloadTest" /> + <Message Text="Extracting CoreFX tests..." Importance="high" /> + <Unzip + SourceFiles="$(MSBuildProjectDirectory)/corefx/tests/$([System.IO.Path]::GetFileName('%(AllTests.Identity)'))" + DestinationFolder="$(MSBuildProjectDirectory)/corefx/tests/extracted/$([System.IO.Path]::GetFileNameWithoutExtension('%(AllTests.Identity)'))" /> + </Target> + +</Project> diff --git a/netcore/dl-test-assets.py b/netcore/dl-test-assets.py deleted file mode 100755 index d422801b513..00000000000 --- a/netcore/dl-test-assets.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import subprocess -import xml.etree.ElementTree as ET -import zipfile -import urllib.request, urllib.parse, urllib.error -import multiprocessing - -if len(sys.argv) < 4: - print("Usage: dl-test-assets.py <path to assets.xml> <base url> <output dir>") - sys.exit(1) - -infile_name = sys.argv [1] -base_url = sys.argv [2] -outdir = sys.argv [3] -tree = ET.parse(infile_name) -root = tree.getroot() - -if not os.path.exists(outdir): - os.makedirs(outdir) - -def downloadAsset(elem): - if elem.tag != "Blob": - return - id = elem.attrib ["Id"] - filename = os.path.basename(id) - print("Downloading " + filename) - try: - name, hdrs = urllib.request.urlretrieve(base_url + "/" + id, outdir + "/" + filename) - except IOError as e: - print("Download failed for " + id) - sys.exit (1) - print("Extracting " + filename) - with zipfile.ZipFile(outdir + "/" + filename) as zf: - zf.extractall(outdir + "/extracted/" + filename[:-4]) - -pool = multiprocessing.Pool(multiprocessing.cpu_count()) -results = pool.map(downloadAsset, root) -pool.close() -pool.join() |