1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
parameters:
steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo)
variables: [] # optional -- list of additional variables to send to the template
jobName: '' # required -- job name
displayName: '' # optional -- display name for the job. Will use jobName if not passed
pool: '' # required -- name of the Build pool
container: '' # required -- name of the container
buildConfig: '' # required -- build configuration
archType: '' # required -- targeting CPU architecture
osGroup: '' # required -- operating system for the job
osSubgroup: '' # optional -- operating system subgroup
extraSetupParameters: '' # optional -- extra arguments to pass to the setup script
frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against
continueOnError: 'false' # optional -- determines whether to continue the build if the step errors
dependsOn: '' # optional -- dependencies of the job
timeoutInMinutes: 320 # optional -- timeout for the job
enableTelemetry: false # optional -- enable for telemetry
liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run
runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono
codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono
projectFile: '' # required -- project file to build helix workitems
runKind: '' # required -- test category
additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific
jobs:
- template: xplat-pipeline-job.yml
parameters:
dependsOn: ${{ parameters.dependsOn }}
buildConfig: ${{ parameters.buildConfig }}
archType: ${{ parameters.archType }}
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup }}
liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
enableTelemetry: ${{ parameters.enableTelemetry }}
enablePublishBuildArtifacts: true
continueOnError: ${{ parameters.continueOnError }}
${{ if ne(parameters.displayName, '') }}:
displayName: '${{ parameters.displayName }}'
${{ if eq(parameters.displayName, '') }}:
displayName: '${{ parameters.jobName }}'
name: ${{ replace(replace(parameters.jobName, ' ', '_'), '-', '_') }}
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
variables:
- ${{ each variable in parameters.variables }}:
- ${{ if ne(variable.name, '') }}:
- name: ${{ variable.name }}
value: ${{ variable.value }}
- ${{ if ne(variable.group, '') }}:
- group: ${{ variable.group }}
- IsInternal: ''
- HelixApiAccessToken: ''
- SharedHelixPreCommands: ''
- AdditionalHelixPreCommands: ''
- AdditionalHelixPostCommands: ''
# run machine-setup and set PYTHONPATH for both public and private jobs
- ${{ if eq(parameters.osGroup, 'windows') }}:
- SharedHelixPreCommands: 'call %HELIX_WORKITEM_PAYLOAD%\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%'
- ${{ if ne(parameters.osGroup, 'windows') }}:
- SharedHelixPreCommands: 'chmod +x $HELIX_WORKITEM_PAYLOAD/machine-setup.sh;. $HELIX_WORKITEM_PAYLOAD/machine-setup.sh;export PYTHONPATH=$HELIX_WORKITEM_PAYLOAD/scripts:$HELIX_WORKITEM_PAYLOAD'
- ${{ if eq(parameters.osGroup, 'windows') }}:
- HelixPreCommandWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install -U pip;py -3 -m pip install --user azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install --user azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
- HelixPostCommandsWindows: 'set PYTHONPATH=%ORIGPYPATH%'
- ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'OSX'), ne(parameters.osSubGroup, '_musl')) }}:
- HelixPreCommandLinux: 'export ORIGPYPATH=$PYTHONPATH;export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- HelixPostCommandsLinux: 'export PYTHONPATH=$ORIGPYPATH'
- ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'OSX'), eq(parameters.osSubGroup, '_musl')) }}:
- HelixPreCommandMusl: 'export ORIGPYPATH=$PYTHONPATH;sudo apk add py3-virtualenv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- HelixPostCommandsMusl: 'export PYTHONPATH=$ORIGPYPATH'
- ${{ if eq(parameters.osGroup, 'OSX') }}:
- HelixPreCommandOSX: 'export ORIGPYPATH=$PYTHONPATH;export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0 --force-reinstall;pip3 install azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- HelixPostCommandOSX: 'export PYTHONPATH=$ORIGPYPATH'
# extra private job settings
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq(parameters.osGroup, 'windows') }}:
- AdditionalHelixPreCommands: $(HelixPreCommandWindows)
- AdditionalHelixPostCommands: $(HelixPostCommandsWindows)
- IsInternal: -Internal
- ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'OSX'), ne(parameters.osSubGroup, '_musl')) }}:
- AdditionalHelixPreCommands: $(HelixPreCommandLinux)
- AdditionalHelixPostCommands: $(HelixPostCommandsLinux)
- IsInternal: --internal
- ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'OSX'), eq(parameters.osSubGroup, '_musl')) }}:
- AdditionalHelixPreCommands: $(HelixPreCommandMusl)
- AdditionalHelixPostCommands: $(HelixPostCommandsMusl)
- IsInternal: --internal
- ${{ if eq(parameters.osGroup, 'OSX') }}:
- AdditionalHelixPreCommands: $(HelixPreCommandOSX)
- AdditionalHelixPostCommands: $(HelixPostCommandOSX)
- IsInternal: --internal
- group: DotNet-HelixApi-Access
- group: dotnet-benchview
- ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
- ${{ if eq(parameters.osGroup, 'windows') }}:
- AdditionalHelixPreCommands: $(HelixPreCommandWindows)
- AdditionalHelixPostCommands: $(HelixPostCommandsWindows)
- ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.osGroup, 'OSX')) }}:
- AdditionalHelixPreCommands: $(HelixPreCommandLinux)
- AdditionalHelixPostCommands: $(HelixPostCommandsLinux)
- ${{ if eq(parameters.osGroup, 'OSX') }}:
- AdditionalHelixPreCommands: $(HelixPreCommandOSX)
- AdditionalHelixPostCommands: $(HelixPostCommandOSX)
- ${{ if ne(parameters.runtimeType, 'wasm') }}:
- ExtraSetupArguments: --install-dir $(PayloadDirectory)/dotnet
- ${{ if and(eq(parameters.runtimeType, 'wasm'), in(variables['Build.Reason'], 'PullRequest')) }}:
- ExtraSetupArguments: ''
workspace:
clean: all
pool:
${{ parameters.pool }}
${{ if ne(parameters.runtimeType, 'wasm') }}:
container: ${{ parameters.container }}
strategy:
matrix:
${{ each framework in parameters.frameworks }}:
${{ framework }}:
_Framework: ${{ framework }}
steps:
- ${{ parameters.steps }}
# run performance-setup
- powershell: $(Build.SourcesDirectory)\eng\testing\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) -Kind ${{ parameters.runKind }} -LogicalMachine ${{ parameters.logicalMachine }} ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }}
displayName: Performance Setup (Windows)
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
- script: $(Build.SourcesDirectory)/eng/testing/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) --kind ${{ parameters.runKind }} --logicalmachine ${{ parameters.logicalMachine }} ${{ parameters.extraSetupParameters }} ${{ parameters.additionalSetupParameters }}
displayName: Performance Setup (Linux/MAC)
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
# run ci-setup
- script: $(Python) $(PerformanceDirectory)\scripts\ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)\machine-setup.cmd
displayName: Run ci setup script (Windows)
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)/machine-setup.sh
displayName: Run ci setup script (Linux/MAC)
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
# copy wasm packs if running on wasm
- script: >-
mkdir -p $(librariesDownloadDir)/bin/wasm/data &&
cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-workload $(librariesDownloadDir)/bin/wasm &&
cp src/mono/wasm/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js &&
find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \;
displayName: "Create wasm directory (Linux)"
condition: and(succeeded(), eq('${{ parameters.runtimeType }}', 'wasm'))
# copy scenario support files
- script: xcopy $(PerformanceDirectory)\scripts $(WorkItemDirectory)\scripts\/e && xcopy $(PerformanceDirectory)\src\scenarios\shared $(WorkItemDirectory)\shared\/e && xcopy $(PerformanceDirectory)\src\scenarios\staticdeps $(WorkItemDirectory)\staticdeps\/e
displayName: Copy scenario support files (Windows)
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- script: cp -r $(PerformanceDirectory)/scripts $(WorkItemDirectory)/scripts/ && cp -r $(PerformanceDirectory)/src/scenarios/shared $(WorkItemDirectory)/shared/ && cp -r $(PerformanceDirectory)/src/scenarios/staticdeps/ $(WorkItemDirectory)/staticdeps/
displayName: Copy scenario support files (Linux/MAC)
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
# build Startup
- script: $(PayloadDirectory)\dotnet\dotnet.exe publish -c Release -o $(WorkItemDirectory)\Startup -f net6.0 -r win-$(Architecture) $(PerformanceDirectory)\src\tools\ScenarioMeasurement\Startup\Startup.csproj
displayName: Build Startup tool (Windows)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/startup -f net6.0 -r linux-$(Architecture) $(PerformanceDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj
displayName: Build Startup tool (Linux)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux'))
- script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/startup -f net6.0 -r osx-$(Architecture) $(PerformanceDirectory)/src/tools/ScenarioMeasurement/Startup/Startup.csproj
displayName: Build Startup tool (MAC)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Darwin'))
# build SizeOnDisk
- script: $(PayloadDirectory)\dotnet\dotnet.exe publish -c Release -o $(WorkItemDirectory)\SOD -f net6.0 -r win-$(Architecture) $(PerformanceDirectory)\src\tools\ScenarioMeasurement\SizeOnDisk\SizeOnDisk.csproj
displayName: Build SizeOnDisk tool (Windows)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/SOD -f net6.0 -r linux-$(Architecture) $(PerformanceDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj
displayName: Build SizeOnDisk tool (Linux)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux'))
- script: $(PayloadDirectory)/dotnet/dotnet publish -c Release -o $(WorkItemDirectory)/SOD -f net6.0 -r osx-$(Architecture) $(PerformanceDirectory)/src/tools/ScenarioMeasurement/SizeOnDisk/SizeOnDisk.csproj
displayName: Build SizeOnDisk tool (MAC)
env:
PERFLAB_TARGET_FRAMEWORKS: net6.0
condition: and(succeeded(), eq(variables['Agent.Os'], 'Darwin'))
# Zip the workitem directory (for xharness based workitems)
- task: ArchiveFiles@2
inputs:
rootFolderOrFile: '$(WorkItemDirectory)'
includeRootFolder: false
archiveFile: '$(WorkItemDirectory).zip'
verbose: True
# run perf testing in helix
- template: /eng/pipelines/coreclr/templates/perf-send-to-helix.yml
parameters:
HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)'
HelixAccessToken: $(HelixApiAccessToken)
HelixTargetQueues: $(Queue)
HelixPreCommands: '$(AdditionalHelixPreCommands);$(SharedHelixPreCommands)' # $(HelixPreCommands) should follow $(AdditionalHelixPreCommands) because PYTHONPATH is cleared by the former
HelixPostCommands: $(AdditionalHelixPostCommands)
Creator: $(Creator)
WorkItemTimeout: 4:00 # 4 hours
WorkItemDirectory: '$(WorkItemDirectory)' # contains scenario tools, shared python scripts, dotnet tool
CorrelationPayloadDirectory: '$(PayloadDirectory)' # contains performance repo and built product
ProjectFile: ${{ parameters.projectFile }}
osGroup: ${{ parameters.osGroup }}
# publish logs
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.runKind }}_$(iOSLlvmBuild)'
continueOnError: true
condition: always()
|
