Convert crossbuild to mounted docker volume format, and always run cleanup (#22122)

tasks

2 files changed, 113 insertions, 164 deletions

diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json b/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json
index d5568222d5..a189c04748 100644
--- a/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json
+++ b/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json
@@ -4,23 +4,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Delete files from $(Build.SourcesDirectory)",
-      "timeoutInMinutes": 0,
-      "task": {
-        "id": "b7e8b412-0437-4065-9371-edc5881de25b",
-        "versionSpec": "1.*",
-        "definitionType": "task"
-      },
-      "inputs": {
-        "SourceFolder": "$(Build.SourcesDirectory)",
-        "Contents": "**\n.gitignore"
-      }
-    },
-    {
-      "enabled": true,
-      "continueOnError": false,
-      "alwaysRun": false,
-      "displayName": "git clone",
+      "displayName": "Change permissions to agent folder for cleanup steps",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -28,9 +12,9 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "git",
-        "arguments": "clone $(PB_VsoCorefxGitUrl) corefx",
-        "workingFolder": "",
+        "filename": "sudo",
+        "arguments": "chmod 777 -R .",
+        "workingFolder": "$(Agent.WorkFolder)",
         "failOnStandardError": "false"
       }
     },
@@ -38,43 +22,23 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "git checkout",
+      "displayName": "Delete files from $(PB_GitDirectory)",
       "timeoutInMinutes": 0,
       "task": {
-        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
-        "versionSpec": "1.*",
-        "definitionType": "task"
-      },
-      "inputs": {
-        "filename": "git",
-        "arguments": "checkout $(SourceVersion)",
-        "workingFolder": "$(Build.SourcesDirectory)/corefx",
-        "failOnStandardError": "false"
-      }
-    },
-    {
-      "enabled": true,
-      "continueOnError": false,
-      "alwaysRun": true,
-      "displayName": "Initialize tools",
-      "timeoutInMinutes": 0,
-      "task": {
-        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
+        "id": "b7e8b412-0437-4065-9371-edc5881de25b",
         "versionSpec": "1.*",
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corefx/init-tools.sh",
-        "arguments": "",
-        "workingFolder": "",
-        "failOnStandardError": "false"
+        "SourceFolder": "$(PB_GitDirectory)",
+        "Contents": "**\n.gitignore\n.editorconfig\n.gitattributes\n.gitmirrorall\n.git/**\n.git"
       }
     },
     {
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Initialize Docker",
+      "displayName": "git clone",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -82,8 +46,8 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corefx/Tools/scripts/docker/init-docker.sh",
-        "arguments": "$(PB_DockerImageName)",
+        "filename": "git",
+        "arguments": "clone $(PB_VsoCorefxGitUrl) $(PB_GitDirectory)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -92,7 +56,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Start detached docker container",
+      "displayName": "git checkout",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -100,9 +64,9 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "docker",
-        "arguments": "run --privileged -d -w $(PB_GitDirectory) --name $(PB_DockerContainerName) $(PB_DockerImageName) sleep 7200",
-        "workingFolder": "",
+        "filename": "git",
+        "arguments": "checkout $(SourceVersion)",
+        "workingFolder": "$(PB_GitDirectory)",
         "failOnStandardError": "false"
       }
     },
@@ -110,7 +74,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Clone repository",
+      "displayName": "Initialize tools",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -118,9 +82,9 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "docker",
-        "arguments": "exec $(PB_DockerContainerName) git clone $(PB_VsoCorefxGitUrl) $(PB_GitDirectory)",
-        "workingFolder": "",
+        "filename": "$(PB_GitDirectory)/init-tools.sh",
+        "arguments": "",
+        "workingFolder": "$(PB_GitDirectory)",
         "failOnStandardError": "false"
       }
     },
@@ -128,7 +92,7 @@
       "enabled": true,
       "continueOnError": false,
       "alwaysRun": false,
-      "displayName": "Check out the specified commit",
+      "displayName": "Initialize Docker",
       "timeoutInMinutes": 0,
       "task": {
         "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
@@ -136,8 +100,8 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "docker",
-        "arguments": "exec $(PB_DockerContainerName) git checkout $(SourceVersion)",
+        "filename": "$(PB_GitDirectory)/Tools/scripts/docker/init-docker.sh",
+        "arguments": "$(PB_DockerImageName)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -155,7 +119,7 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "exec $(PB_DockerContainerName) $(PB_GitDirectory)/build-managed.sh -OfficialBuildId=$(OfficialBuildId) -- /t:GenerateVersionSourceFile /p:GenerateVersionSourceFile=true",
+        "arguments": "run $(PB_DockerCommonRunArgs) $(PB_DockerVolumeName)/build-managed.sh -OfficialBuildId=$(OfficialBuildId) -- /t:GenerateVersionSourceFile /p:GenerateVersionSourceFile=true",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -173,7 +137,7 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "exec $(PB_DockerContainerName) $(PB_GitDirectory)/sync.sh -p -- /p:ArchGroup=$(PB_Architecture)",
+        "arguments": "run $(PB_DockerCommonRunArgs) $(PB_DockerVolumeName)/sync.sh -p -- /p:ArchGroup=$(PB_Architecture)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -191,7 +155,7 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "exec -e ROOTFS_DIR $(PB_DockerContainerName) $(PB_GitDirectory)/build.sh -OfficialBuildId=$(OfficialBuildId) $(PB_BuildArguments)",
+        "arguments": "run -e ROOTFS_DIR $(PB_DockerCommonRunArgs) $(PB_DockerVolumeName)/build.sh -OfficialBuildId=$(OfficialBuildId) $(PB_BuildArguments)",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -209,59 +173,7 @@
       },
       "inputs": {
         "filename": "docker",
-        "arguments": "exec $(PB_DockerContainerName) $(PB_GitDirectory)/publish-packages.sh -AzureAccount=$(PB_CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(PB_Label) -verbose -- /p:OverwriteOnPublish=false",
-        "workingFolder": "",
-        "failOnStandardError": "false"
-      }
-    },
-    {
-      "enabled": true,
-      "continueOnError": true,
-      "alwaysRun": true,
-      "displayName": "Remove old docker build logs",
-      "timeoutInMinutes": 0,
-      "task": {
-        "id": "b7e8b412-0437-4065-9371-edc5881de25b",
-        "versionSpec": "1.*",
-        "definitionType": "task"
-      },
-      "inputs": {
-        "SourceFolder": "$(PB_DockerCopyDest)",
-        "Contents": "*"
-      }
-    },
-    {
-      "enabled": true,
-      "continueOnError": true,
-      "alwaysRun": true,
-      "displayName": "Expose docker repo for publishing",
-      "timeoutInMinutes": 0,
-      "task": {
-        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
-        "versionSpec": "1.*",
-        "definitionType": "task"
-      },
-      "inputs": {
-        "filename": "docker",
-        "arguments": "cp $(PB_DockerContainerName):$(PB_GitDirectory) $(PB_DockerCopyDest)",
-        "workingFolder": "",
-        "failOnStandardError": "false"
-      }
-    },
-    {
-      "enabled": true,
-      "continueOnError": true,
-      "alwaysRun": true,
-      "displayName": "Remove container",
-      "timeoutInMinutes": 0,
-      "task": {
-        "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9",
-        "versionSpec": "1.*",
-        "definitionType": "task"
-      },
-      "inputs": {
-        "filename": "docker",
-        "arguments": "rm -f $(PB_DockerContainerName)",
+        "arguments": "run $(PB_DockerCommonRunArgs) $(PB_DockerVolumeName)/publish-packages.sh -AzureAccount=$(PB_CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(PB_Label) -verbose -- /p:OverwriteOnPublish=false",
         "workingFolder": "",
         "failOnStandardError": "false"
       }
@@ -278,7 +190,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "CopyRoot": "$(PB_DockerCopyDest)/corefx",
+        "CopyRoot": "$(PB_GitDirectory)",
         "Contents": "*.log",
         "ArtifactName": "BuildLogs",
         "ArtifactType": "Container",
@@ -297,7 +209,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corefx/Tools/scripts/docker/cleanup-docker.sh",
+        "filename": "$(PB_GitDirectory)/Tools/scripts/docker/cleanup-docker.sh",
         "arguments": "",
         "workingFolder": "",
         "failOnStandardError": "false"
@@ -315,7 +227,7 @@
         "definitionType": "task"
       },
       "inputs": {
-        "filename": "$(Build.SourcesDirectory)/corefx/Tools/msbuild.sh",
+        "filename": "$(PB_GitDirectory)/Tools/msbuild.sh",
         "arguments": "cleanupagent.proj /p:AgentDirectory=$(Agent.HomeDirectory) /p:DoClean=$(PB_CleanAgent)",
         "workingFolder": "$(Build.SourcesDirectory)/corefx/Tools/scripts/vstsagent/",
         "failOnStandardError": "false"
@@ -326,6 +238,25 @@
     {
       "enabled": false,
       "definition": {
+        "id": "5d58cc01-7c75-450c-be18-a388ddb129ec"
+      },
+      "inputs": {
+        "branchFilters": "[\"+refs/heads/*\"]",
+        "additionalFields": "{}"
+      }
+    },
+    {
+      "enabled": false,
+      "definition": {
+        "id": "5bc3cfb7-6b54-4a4b-b5d2-a3905949f8a6"
+      },
+      "inputs": {
+        "additionalFields": "{}"
+      }
+    },
+    {
+      "enabled": false,
+      "definition": {
         "id": "7c555368-ca64-4199-add6-9ebaf0b0137d"
       },
       "inputs": {
@@ -357,22 +288,40 @@
     }
   ],
   "variables": {
-    "system.debug": {
-      "value": "false",
-      "allowOverride": true
+    "CloudDropAccessToken": {
+      "value": null,
+      "isSecret": true
+    },
+    "GitHubBranch": {
+      "value": "sni_plus_latestbuildtools"
+    },
+    "OfficialBuildId": {
+      "value": "$(Build.BuildNumber)"
+    },
+    "PB_Architecture": {
+      "value": "arm"
+    },
+    "PB_BuildArguments": {
+      "value": "-BuildArch=$(PB_Architecture)"
+    },
+    "PB_CleanAgent": {
+      "value": "true"
+    },
+    "PB_CloudDropAccountName": {
+      "value": "dotnetbuildoutput"
     },
     "PB_ConfigurationGroup": {
       "value": "Release"
     },
-    "PB_VsoCorefxGitUrl": {
-      "value": "http://github.com/dotnet/corefx.git"
-    },
-    "PB_GitDirectory": {
-      "value": "/root/corefx"
+    "PB_DockerCommonRunArgs": {
+      "value": "--rm --name $(PB_DockerContainerName) -v \"$(PB_GitDirectory):$(PB_DockerVolumeName)\" -w=\"$(PB_DockerVolumeName)\" $(PB_DockerImageName)"
     },
     "PB_DockerContainerName": {
       "value": "corefx-cross-$(Build.BuildId)"
     },
+    "PB_DockerCopyDest": {
+      "value": "$(Build.BinariesDirectory)/docker_repo"
+    },
     "PB_DockerImageName": {
       "value": "$(PB_DockerRepository):$(PB_DockerTag)"
     },
@@ -380,47 +329,45 @@
       "value": "microsoft/dotnet-buildtools-prereqs"
     },
     "PB_DockerTag": {
-      "value": "ubuntu1404_cross_prereqs_v2",
+      "value": "ubuntu-14.04-cross-0cd4667-20172211042239",
       "allowOverride": true
     },
-    "PB_CloudDropAccountName": {
-      "value": "dotnetbuildoutput"
-    },
-    "CloudDropAccessToken": {
-      "value": null,
-      "isSecret": true
+    "PB_DockerVolumeName": {
+      "value": "/root/corefx-$(Build.BuildId)"
     },
-    "OfficialBuildId": {
-      "value": "$(Build.BuildNumber)"
+    "PB_GitDirectory": {
+      "value": "$(Build.SourcesDirectory)/corefx"
     },
     "PB_Label": {
       "value": "$(Build.BuildNumber)"
     },
-    "GitHubBranch": {
-      "value": "sni_plus_latestbuildtools"
+    "PB_VsoAccountName": {
+      "value": "dn-bot"
     },
-    "PB_Architecture": {
-      "value": "arm"
+    "PB_VsoCorefxGitUrl": {
+      "value": "https://$(PB_VsoAccountName):$(VsoPassword)@devdiv.visualstudio.com/DevDiv/_git/$(PB_VsoRepositoryName)/"
     },
-    "PB_BuildArguments": {
+    "PB_VsoRepositoryName": {
+      "value": "DotNet-CoreFX-Trusted"
+    },
+    "portableBuild": {
       "value": ""
     },
+    "ROOTFS_DIR": {
+      "value": "/crossrootfs/$(PB_Architecture)",
+      "allowOverride": true
+    },
     "SourceVersion": {
       "value": "HEAD",
       "allowOverride": true
     },
-    "PB_DockerCopyDest": {
-      "value": "$(Build.BinariesDirectory)/docker_repo"
-    },
-    "ROOTFS_DIR": {
-      "value": "/crossrootfs/$(PB_Architecture)",
+    "system.debug": {
+      "value": "false",
       "allowOverride": true
     },
-    "portableBuild": {
-      "value": ""
-    },
-    "PB_CleanAgent": {
-      "value": "true"
+    "VsoPassword": {
+      "value": null,
+      "isSecret": true
     }
   },
   "demands": [
@@ -445,6 +392,7 @@
   "buildNumberFormat": "$(date:yyyyMMdd)$(rev:-rr)",
   "jobAuthorizationScope": "projectCollection",
   "jobTimeoutInMinutes": 90,
+  "jobCancelTimeoutInMinutes": 5,
   "repository": {
     "properties": {
       "labelSources": "0",
@@ -452,7 +400,9 @@
       "fetchDepth": "0",
       "gitLfsSupport": "false",
       "skipSyncSource": "false",
-      "cleanOptions": "0"
+      "cleanOptions": "0",
+      "checkoutNestedSubmodules": "false",
+      "labelSourcesFormat": "$(build.buildNumber)"
     },
     "id": "58fa2458-e392-4373-ba2b-dd3ef0c2d7ce",
     "type": "TfsGit",
@@ -462,26 +412,27 @@
     "clean": "false",
     "checkoutSubmodules": false
   },
+  "processParameters": {},
   "quality": "definition",
   "queue": {
+    "id": 36,
+    "name": "DotNet-Build",
     "pool": {
       "id": 39,
       "name": "DotNet-Build"
-    },
-    "id": 36,
-    "name": "DotNet-Build"
+    }
   },
-  "path": "\\",
-  "type": "build",
   "id": 5247,
   "name": "DotNet-CoreFx-Trusted-Linux-Crossbuild",
-  "url": "https://devdiv.visualstudio.com/DefaultCollection/0bdbc590-a062-4c3f-b0f6-9383f67865ee/_apis/build/Definitions/5247",
+  "path": "\\",
+  "type": "build",
   "project": {
     "id": "0bdbc590-a062-4c3f-b0f6-9383f67865ee",
     "name": "DevDiv",
     "description": "Visual Studio and DevDiv team project for git source code repositories.  Work items will be added for Adams, Dev14 work items are tracked in vstfdevdiv.  ",
     "url": "https://devdiv.visualstudio.com/DefaultCollection/_apis/projects/0bdbc590-a062-4c3f-b0f6-9383f67865ee",
     "state": "wellFormed",
-    "revision": 418097503
+    "revision": 418097767,
+    "visibility": "private"
   }
 }
\ No newline at end of file
diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Linux.json b/buildpipeline/DotNet-CoreFx-Trusted-Linux.json
index 769eb89364..e78b77a7a7 100644
--- a/buildpipeline/DotNet-CoreFx-Trusted-Linux.json
+++ b/buildpipeline/DotNet-CoreFx-Trusted-Linux.json
@@ -217,7 +217,7 @@
     {
       "enabled": true,
       "continueOnError": true,
-      "alwaysRun": false,
+      "alwaysRun": true,
       "displayName": "Copy Publish Artifact: BuildLogs",
       "timeoutInMinutes": 0,
       "task": {
@@ -236,7 +236,7 @@
     {
       "enabled": true,
       "continueOnError": true,
-      "alwaysRun": false,
+      "alwaysRun": true,
       "displayName": "Cleanup Docker",
       "timeoutInMinutes": 0,
       "task": {
@@ -254,7 +254,7 @@
     {
       "enabled": true,
       "continueOnError": true,
-      "alwaysRun": false,
+      "alwaysRun": true,
       "displayName": "Cleanup VSTS Agent",
       "timeoutInMinutes": 0,
       "task": {
@@ -433,7 +433,7 @@
       "deleteTestResults": true
     }
   ],
-  "buildNumberFormat": "$(date:yyyyMMdd)$(rev:-9r)",
+  "buildNumberFormat": "$(date:yyyyMMdd)$(rev:-rr)",
   "jobAuthorizationScope": "projectCollection",
   "jobTimeoutInMinutes": 60,
   "jobCancelTimeoutInMinutes": 5,
@@ -444,9 +444,7 @@
       "fetchDepth": "0",
       "gitLfsSupport": "false",
       "skipSyncSource": "true",
-      "cleanOptions": "0",
-      "checkoutNestedSubmodules": "false",
-      "labelSourcesFormat": "$(build.buildNumber)"
+      "cleanOptions": "0"
     },
     "id": "58fa2458-e392-4373-ba2b-dd3ef0c2d7ce",
     "type": "TfsGit",