1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
|
# default-build.yml
# Description: Defines a build phase for invoking build.sh/cmd
# Parameters:
# jobName: string
# The name of the job. Defaults to the name of the OS. No spaces allowed
# jobDisplayName: string
# The friendly job name to display in the UI. Defaults to the name of the OS.
# agentOs: string
# Used in templates to define variables which are OS specific. Typically from the set { Windows, Linux, macOS }
# buildArgs: string
# Additional arguments to pass to the build.sh/cmd script.
# Note: -ci is always passed
# beforeBuild: [steps]
# Additional steps to run before build.sh/cmd
# steps: [steps]
# Instead of running build.cmd/sh, run these build steps.
# afterBuild: [steps]
# Additional steps to run after build.sh/cmd
# artifacts: [array]
# name: string
# The name of the artifact container
# - path: string
# The file path to artifacts output
# includeForks: boolean
# Should artifacts from forks be published?
# publishOnError: boolean
# Should artifacts be published if previous step failed?
# dependsOn: string | [ string ]
# For fan-out/fan-in. https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#job
# condition: string
# A condition which can be used to skip the job completely
# codeSign: boolean
# This build definition is enabled for code signing. (Only applies to Windows)
# buildDirectory: string
# Specifies what directory to run build.sh/cmd
# skipComponentGovernanceDetection: boolean
# Determines if component governance detection can be skipped
#
# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details
#
parameters:
agentOs: 'Windows'
buildArgs: ''
configuration: 'Release'
beforeBuild: []
# steps: [] don't define an empty object default because there is no way in template expression yet to check "if isEmpty(parameters.steps)"
afterBuild: []
codeSign: false
dependsOn: ''
condition: ''
# jobName: '' - use agentOs by default.
# jobDisplayName: '' - use agentOs by default.
artifacts: []
buildDirectory: $(System.DefaultWorkingDirectory)/eng/
installTar: true
installNodeJs: true
installJdk: true
timeoutInMinutes: 180
testRunTitle: $(AgentOsName)-$(BuildConfiguration)
useHostedUbuntu: true
skipComponentGovernanceDetection: false
# We need longer than the default amount of 5 minutes to upload our logs/artifacts. (We currently take around 5 mins in the best case).
# This makes sure we have time to upload everything in the case of a build timeout - really important for investigating a build
# timeout due to test hangs.
cancelTimeoutInMinutes: 15
jobs:
- template: /eng/common/templates/job/job.yml
parameters:
name: ${{ coalesce(parameters.jobName, parameters.agentOs) }}
displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }}
dependsOn: ${{ parameters.dependsOn }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
${{ if ne(parameters.enableRichCodeNavigation, '') }}:
enableRichCodeNavigation: true
richCodeNavigationLanguage: 'csharp,typescript,java'
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
enableMicrobuild: true
enablePublishBuildAssets: true
enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }}
enablePublishTestResults: ${{ eq(parameters.isTestingJob, 'true') }} # publish test results to AzDO (populates AzDO Tests tab)
mergeTestResults: true
testRunTitle: ${{ parameters.testRunTitle }}
enableTelemetry: true
helixRepo: dotnet/aspnetcore
helixType: build.product/
workspace:
clean: all
# Map friendly OS names to the right queue
# See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md
pool:
${{ if eq(parameters.agentOs, 'macOS') }}:
vmImage: macOS-10.15
${{ if eq(parameters.agentOs, 'Linux') }}:
${{ if eq(parameters.useHostedUbuntu, true) }}:
vmImage: ubuntu-18.04
${{ if eq(parameters.useHostedUbuntu, false) }}:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Svc-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
${{ if eq(parameters.agentOs, 'Windows') }}:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Svc-Public
${{ if ne(parameters.isTestingJob, true) }}:
# Visual Studio Build Tools
demands: ImageOverride -equals Build.Server.Amd64.VS2019.BT.Open
${{ if eq(parameters.isTestingJob, true) }}:
# Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing
demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Svc-Internal
# Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing
demands: ImageOverride -equals Build.Server.Amd64.VS2019
${{ if ne(parameters.container, '') }}:
container: ${{ parameters.container }}
variables:
- AgentOsName: ${{ parameters.agentOs }}
- ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
- BuildScriptArgs: ${{ parameters.buildArgs }}
- _BuildConfig: ${{ parameters.configuration }}
- BuildConfiguration: ${{ parameters.configuration }}
- ${{ if eq(parameters.buildDirectory, '') }}:
- BuildDirectory: $(System.DefaultWorkingDirectory)
- ${{ if ne(parameters.buildDirectory, '') }}:
- BuildDirectory: ${{ parameters.buildDirectory }}
- DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
- DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
- TeamName: AspNetCore
- ${{ if eq(parameters.agentOs, 'Linux') }}:
- LC_ALL: 'en_US.UTF-8'
- LANG: 'en_US.UTF-8'
- LANGUAGE: 'en_US.UTF-8'
- ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
- JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
- ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}:
- _SignType: ''
- ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}:
- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
- _SignType: real
- ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
- _SignType: test
- LC_ALL: 'en_US.UTF-8'
- LANG: 'en_US.UTF-8'
- LANGUAGE: 'en_US.UTF-8'
steps:
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- script: df -h
displayName: Disk size
- ${{ if eq(parameters.agentOs, 'macOS') }}:
- script: sudo xcode-select -s /Applications/Xcode_12.2.app/Contents/Developer
displayName: Use XCode 12.2
- checkout: self
clean: true
- ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isTestingJob, true)) }}:
- powershell: ./eng/scripts/InstallProcDump.ps1
displayName: Install ProcDump
- powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet
displayName: Start background dump collection
- ${{ if eq(parameters.installNodeJs, 'true') }}:
- task: NodeTool@0
displayName: Install Node 14.x
inputs:
versionSpec: 14.x
- ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
- powershell: ./eng/scripts/InstallJdk.ps1
displayName: Install JDK 11
- ${{ if eq(parameters.isTestingJob, true) }}:
- powershell: |
Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
./eng/scripts/InstallGoogleChrome.ps1
displayName: Install Chrome
- ${{ if and(eq(parameters.installTar, 'true'), eq(parameters.agentOs, 'Windows')) }}:
- powershell: ./eng/scripts/InstallTar.ps1
displayName: Find or install Tar
- ${{ if eq(parameters.agentOs, 'Windows') }}:
- powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
displayName: Add dotnet tools to path
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools"
displayName: Add dotnet tools to path
- ${{ parameters.beforeBuild }}
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- ${{ if eq(parameters.agentOs, 'Windows') }}:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- task: PowerShell@2
displayName: Setup Private Feeds Credentials
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- task: Bash@3
displayName: Setup Private Feeds Credentials
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)
# Add COMPlus_* environment variables to build steps.
- ${{ if ne(parameters.steps, '')}}:
- ${{ each step in parameters.steps }}:
# Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`.
# Aim here is to avoid having two `env:` properties in the expanded YAML.
- ${{ each pair in step }}:
${{ if ne(pair.key, 'env') }}:
${{ pair.key }}: ${{ pair.value }}
env:
# Include the variables we always want.
COMPlus_DbgEnableMiniDump: 1
COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
DotNetCliMsrcReadSasToken: "$(dotnetclimsrc-read-sas-token)"
# Expand provided `env:` properties, if any.
${{ if step.env }}:
${{ step.env }}
- ${{ if eq(parameters.steps, '')}}:
- ${{ if eq(parameters.agentOs, 'Windows') }}:
- script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType)
displayName: Run build.cmd
env:
COMPlus_DbgEnableMiniDump: 1
COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
DotNetCliMsrcReadSasToken: "$(dotnetclimsrc-read-sas-token)"
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
displayName: Run build.sh
env:
COMPlus_DbgEnableMiniDump: 1
COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
DotNetCliMsrcReadSasToken: "$(dotnetclimsrc-read-sas-token)"
- ${{ parameters.afterBuild }}
- ${{ if eq(parameters.agentOs, 'Linux') }}:
- script: df -h && du -h --threshold=50MB ..
displayName: Disk utilization
- ${{ if eq(parameters.agentOs, 'macOS') }}:
- script: df -h && du -h -d 3 ..
displayName: Disk utilization
- ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isTestingJob, true)) }}:
- powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/
displayName: Finish background dump collection
continueOnError: true
condition: always()
- ${{ if eq(parameters.agentOs, 'Windows') }}:
- powershell: eng\scripts\KillProcesses.ps1
displayName: Kill processes
continueOnError: true
condition: always()
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- script: eng/scripts/KillProcesses.sh
displayName: Kill processes
continueOnError: true
condition: always()
# Run component detection after all successful Build:* jobs unless overridden e.g. for Alpine build.
# Make sure auto-injected component detection does _not_ execute in other jobs nor when overridden.
- ${{ if or(not(startsWith(parameters.jobDisplayName, 'Build:')), eq(parameters.skipComponentGovernanceDetection, 'true')) }}:
- script: echo "##vso[task.setvariable variable=CG_RAN]true"
displayName: 'Skip Component Detection'
- ${{ each artifact in parameters.artifacts }}:
- task: PublishBuildArtifacts@1
displayName: Upload artifacts from ${{ artifact.path }}
condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true')))
continueOnError: true
inputs:
pathtoPublish: $(Build.SourcesDirectory)/${{ artifact.path }}
${{ if eq(artifact.name, '') }}:
artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
${{ if ne(artifact.name, '') }}:
artifactName: ${{ artifact.name }}
artifactType: Container
parallel: true
- ${{ if eq(parameters.agentOs, 'Windows') }}:
- powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/
condition: failed()
displayName: Upload cores
- ${{ if ne(parameters.agentOs, 'Windows') }}:
- script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh
condition: failed()
displayName: Upload cores
- ${{ if and(eq(parameters.isTestingJob, true), ne(parameters.jobName, 'Windows_Templates_Test')) }}:
- task: PublishTestResults@2
displayName: Publish js test results
condition: always()
inputs:
testRunner: junit
testResultsFiles: '**/artifacts/log/**/*.junit.xml'
testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js
mergeTestResults: true
buildConfiguration: $(BuildConfiguration)
buildPlatform: $(AgentOsName)
|