Split build scripts for CI (#2309)

In order to sign and package our builds properly in CI we need to split the build phases up. These changes add build-native, build-managed, and build-tests scripts, with build-packages soon to come. Summary:
- Variable handling moved to buildvars-setup scripts to avoid duplication.
- Everything else is from the old scripts with some path cleanup changes.
- Build.cmd/sh now calls into the split scripts.
- Updated build pipeline definitions for new scripts.

3 files changed, 181 insertions, 19 deletions

diff --git a/buildpipeline/DotNet-CoreRT-Linux.json b/buildpipeline/DotNet-CoreRT-Linux.json
index 3053e6299..dab267977 100644
--- a/buildpipeline/DotNet-CoreRT-Linux.json
+++ b/buildpipeline/DotNet-CoreRT-Linux.json
@@ -209,7 +209,7 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/cibuild/clean.sh",
+        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/buildscripts/clean.sh",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -254,7 +254,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Run build.sh",
+      "displayName": "Run build-native.sh",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -263,7 +263,115 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/build.sh $(Platform) $(Configuration)",
+        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/buildscripts/build-native.sh $(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Commit changes",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "commit $(DockerContainerName) $(DockerModifiedImageName)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Remove container",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "rm $(DockerContainerName)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Run build-managed.sh",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/buildscripts/build-managed.sh $(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Commit changes",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "commit $(DockerContainerName) $(DockerModifiedImageName)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Remove container",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "rm $(DockerContainerName)",
+        "workingFolder": "",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Run build-tests.sh",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "docker",
+        "arguments": "run -w=\"$(GitDirectory)\" --name $(DockerContainerName) $(DockerModifiedImageName) $(GitDirectory)/buildscripts/build-tests.sh $(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
diff --git a/buildpipeline/DotNet-CoreRT-Mac.json b/buildpipeline/DotNet-CoreRT-Mac.json
index b0eb5a6eb..b5e725c5c 100644
--- a/buildpipeline/DotNet-CoreRT-Mac.json
+++ b/buildpipeline/DotNet-CoreRT-Mac.json
@@ -56,7 +56,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Run $(Build.SourcesDirectory)/corert/cibuild/clean.sh",
+      "displayName": "Run $(Build.SourcesDirectory)/corert/buildscripts/clean.sh",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -64,7 +64,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corert/cibuild/clean.sh",
+        "filename": "$(Build.SourcesDirectory)/corert/buildscripts/clean.sh",
         "arguments": "-all",
         "workingFolder": "corert",
         "failOnStandardError": "false"
@@ -74,7 +74,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Build",
+      "displayName": "Build native",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -82,8 +82,44 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corert/build.sh",
-        "arguments": "$(Platform) $(Configuration)",
+        "filename": "$(Build.SourcesDirectory)/corert/buildscripts/build-native.sh",
+        "arguments": "$(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
+        "workingFolder": "corert",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Build managed",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "$(Build.SourcesDirectory)/corert/buildscripts/build-managed.sh",
+        "arguments": "$(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
+        "workingFolder": "corert",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Build tests",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "$(Build.SourcesDirectory)/corert/buildscripts/build-tests.sh",
+        "arguments": "$(Platform) $(Configuration) -OfficialBuildId $(OfficialBuildId)",
         "workingFolder": "corert",
         "failOnStandardError": "false"
       }
@@ -100,7 +136,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corert/publish-packages.sh",
+        "filename": "$(Build.SourcesDirectory)/corert/buildscripts/publish-packages.sh",
         "arguments": "",
         "workingFolder": "corert",
         "failOnStandardError": "false"
diff --git a/buildpipeline/DotNet-CoreRT-Windows.json b/buildpipeline/DotNet-CoreRT-Windows.json
index fc7706fd8..2dc2d075b 100644
--- a/buildpipeline/DotNet-CoreRT-Windows.json
+++ b/buildpipeline/DotNet-CoreRT-Windows.json
@@ -94,7 +94,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Run $(Build.SourcesDirectory)\\corert\\cibuild\\clean.cmd",
+      "displayName": "Run $(Build.SourcesDirectory)\\corert\\buildscripts\\clean.cmd",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -102,17 +102,35 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)\\corert\\cibuild\\clean.cmd",
+        "filename": "$(Build.SourcesDirectory)\\corert\\buildscripts\\clean.cmd",
         "arguments": "-all",
         "workingFolder": "corert",
         "failOnStandardError": "false"
       }
     },
     {
-      "enabled": false,
+      "enabled": true,
+      "continueOnError": false,
+      "alwaysRun": false,
+      "displayName": "Build native",
+      "timeoutInMinutes": 0,
+      "task": {
+        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "versionSpec": "*",
+        "definitionType": "task"
+      },
+      "inputs": {
+        "filename": "$(Build.SourcesDirectory)\\corert\\buildscripts\\build-native.cmd",
+        "arguments": "/OfficialBuildId=$(OfficialBuildId) $(Platform) $(Configuration)",
+        "workingFolder": "corert",
+        "failOnStandardError": "false"
+      }
+    },
+    {
+      "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Run $(Build.SourcesDirectory)\\corert\\sync.cmd",
+      "displayName": "Build managed",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -120,8 +138,8 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)\\corert\\sync.cmd",
-        "arguments": "-p",
+        "filename": "$(Build.SourcesDirectory)\\corert\\buildscripts\\build-managed.cmd",
+        "arguments": "/OfficialBuildId=$(OfficialBuildId) $(Platform) $(Configuration)",
         "workingFolder": "corert",
         "failOnStandardError": "false"
       }
@@ -130,7 +148,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Build",
+      "displayName": "Build tests",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -138,8 +156,8 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)\\corert\\build.cmd",
-        "arguments": "$(Platform) $(Configuration)",
+        "filename": "$(Build.SourcesDirectory)\\corert\\buildscripts\\build-tests.cmd",
+        "arguments": "/OfficialBuildId=$(OfficialBuildId) $(Platform) $(Configuration)",
         "workingFolder": "corert",
         "failOnStandardError": "false"
       }
@@ -343,7 +361,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)\\corert\\cibuild\\clean.cmd",
+        "filename": "$(Build.SourcesDirectory)\\corert\\buildscripts\\clean.cmd",
         "arguments": "",
         "workingFolder": "corert",
         "failOnStandardError": "false"