Sync shared code from arcade (#89)

Co-authored-by: dougbu <dougbu@users.noreply.github.com>

9 files changed, 88 insertions, 34 deletions

diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index 42516bb..f163fb9 100644
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma"
 __AndroidPackages+=" krb5"
 __AndroidPackages+=" openssl"
 
-for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
+for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\
     grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
 
     if [[ "$path" != "Filename:" ]]; then
         echo "Working on: $path"
-        wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir"
+        wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir"
     fi
 done
 
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index 65f87b4..7aabaa1 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -46,6 +46,7 @@ jobs:
     - template: /eng/common/templates/variables/sdl-variables.yml
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+    - template: /eng/common/templates/variables/pool-providers.yml
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
     ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
@@ -53,7 +54,7 @@ jobs:
       demands: Cmd
     # If it's not devdiv, it's dnceng
     ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-      name: NetCore1ESPool-Internal
+      name: $(DncEngInternalBuildPool)
       demands: ImageOverride -equals windows.vs2019.amd64
   steps:
   - checkout: self
diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml
index 10e75fc..c2cabcf 100644
--- a/eng/common/templates/job/onelocbuild.yml
+++ b/eng/common/templates/job/onelocbuild.yml
@@ -31,6 +31,17 @@ jobs:
 
   displayName: OneLocBuild${{ parameters.JobNameSuffix }}
 
+  variables:
+    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+    - name: _GenerateLocProjectArguments
+      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+        -LanguageSet "${{ parameters.LanguageSet }}"
+        -CreateNeutralXlfs
+    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+      - name: _GenerateLocProjectArguments
+        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+    - template: /eng/common/templates/variables/pool-providers.yml
+
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
   ${{ if eq(parameters.pool, '') }}:
@@ -41,20 +52,9 @@ jobs:
         demands: Cmd
       # If it's not devdiv, it's dnceng
       ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-        name: NetCore1ESPool-Internal
+        name: $(DncEngInternalBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64
 
-  variables:
-    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
-    - name: _GenerateLocProjectArguments
-      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-        -LanguageSet "${{ parameters.LanguageSet }}"
-        -CreateNeutralXlfs
-    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
-      - name: _GenerateLocProjectArguments
-        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
-      
-
   steps:
     - task: Powershell@2
       inputs:
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index 1cbb6a0..1f1b78f 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -40,9 +40,8 @@ jobs:
   ${{ else }}:
     displayName: Publish to Build Asset Registry
 
-  pool: ${{ parameters.pool }}
-
   variables:
+  - template: /eng/common/templates/variables/pool-providers.yml
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: Publish-Build-Assets
     - group: AzureDevOps-Artifact-Feeds-Pats
@@ -51,6 +50,16 @@ jobs:
     - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
       - template: /eng/common/templates/post-build/common-variables.yml
 
+  pool:
+    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+      name: VSEngSS-MicroBuild2022-1ES
+      demands: Cmd
+    # If it's not devdiv, it's dnceng
+    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+      name: $(DncEngInternalBuildPool)
+      demands: ImageOverride -equals windows.vs2019.amd64
+
   steps:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: DownloadBuildArtifacts@0
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index 88f6f75..6fe2d73 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -34,6 +34,8 @@ parameters:
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
+  variables:
+    - template: /eng/common/templates/variables/pool-providers.yml
 
   ${{ each property in parameters.platform.jobProperties }}:
     ${{ property.key }}: ${{ property.value }}
@@ -46,10 +48,10 @@ jobs:
     # source-build builds run in Docker, including the default managed platform.
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore-Public
+        name: $(DncEngPublicBuildPool)
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: NetCore1ESPool-Internal
+        name: $(DncEngInternalBuildPool)
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
   ${{ if ne(parameters.platform.pool, '') }}:
     pool: ${{ parameters.platform.pool }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
index 21fd122..09c506d 100644
--- a/eng/common/templates/job/source-index-stage1.yml
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -22,16 +22,17 @@ jobs:
     value: ${{ parameters.binlogPath }}
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: source-dot-net stage1 variables
+  - template: /eng/common/templates/variables/pool-providers.yml
 
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
   ${{ if eq(parameters.pool, '') }}:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore-Public
+        name: $(DncEngPublicBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: NetCore1ESPool-Internal
+        name: $(DncEngInternalBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64
 
   steps:
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 64e5929..289bb23 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -88,15 +88,6 @@ jobs:
             - ${{ job.job }}
         - ${{ if eq(parameters.enableSourceBuild, true) }}:
           - Source_Build_Complete
-        pool:
-          # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
-          ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-            name: VSEngSS-MicroBuild2022-1ES
-            demands: Cmd
-          # If it's not devdiv, it's dnceng
-          ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-            name: NetCore1ESPool-Internal
-            demands: ImageOverride -equals windows.vs2019.amd64
 
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 258ed2d..91251d0 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -95,6 +95,7 @@ stages:
     displayName: Validate Build Assets
     variables:
       - template: common-variables.yml
+      - template: /eng/common/templates/variables/pool-providers.yml
     jobs:
     - job:
       displayName: NuGet Validation
@@ -106,7 +107,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
 
       steps:
@@ -143,7 +144,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
@@ -203,7 +204,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
@@ -251,6 +252,7 @@ stages:
     displayName: Publish using Darc
     variables:
       - template: common-variables.yml
+      - template: /eng/common/templates/variables/pool-providers.yml
     jobs:
     - job:
       displayName: Publish Using Darc
@@ -262,7 +264,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml
new file mode 100644
index 0000000..a7b943c
--- /dev/null
+++ b/eng/common/templates/variables/pool-providers.yml
@@ -0,0 +1,48 @@
+# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, 
+# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches.
+
+# Motivation: 
+#   Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS
+#   (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing
+#   (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS.
+#   Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services 
+#   team needs to move resources around and create new and potentially differently-named pools. Using this template 
+#   file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming.
+
+# How to use: 
+#  This yaml assumes your shipped product branches use the naming convention "release/..." (which many do).
+#  If we find alternate naming conventions in broad usage it can be added to the condition below.
+#
+#  First, import the template in an arcade-ified repo to pick up the variables, e.g.:
+#
+#  variables:
+#  - template: eng/common/templates/variables/pool-providers.yml
+#
+#  ... then anywhere specifying the pool provider use the runtime variables,
+#      $(DncEngInternalBuildPool) and $  (DncEngPublicBuildPool), e.g.:
+#
+#        pool:
+#           name: $(DncEngInternalBuildPool)
+#           demands: ImageOverride -equals windows.vs2019.amd64
+
+variables:
+# Coalesce the target and source branches so we know when a PR targets a release branch
+# If these variables are somehow missing, fall back to main (tends to have more capacity)
+- name: BranchNameForPoolSelection
+  value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }}
+
+# Any new -Svc alternative pools should have variables added here to allow for splitting work
+
+# Main branch pools
+- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}:
+  - name: DncEngPublicBuildPool
+    value: NetCore-Public
+  - name: DncEngInternalBuildPool
+    value: NetCore1ESPool-Internal
+
+# Release branch pools
+- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}:
+  - name: DncEngPublicBuildPool
+    value: NetCore-Svc-Public
+  - name: DncEngInternalBuildPool
+    value: NetCore1ESPool-Svc-Internal
\ No newline at end of file