Engineering - Exploration to run tests in parallel (#149698)

4 files changed, 487 insertions, 265 deletions

diff --git a/build/azure-pipelines/darwin/product-build-darwin-test.yml b/build/azure-pipelines/darwin/product-build-darwin-test.yml
index 773b5a40845..dd495426b6d 100644
--- a/build/azure-pipelines/darwin/product-build-darwin-test.yml
+++ b/build/azure-pipelines/darwin/product-build-darwin-test.yml
@@ -1,6 +1,15 @@
 parameters:
   - name: VSCODE_QUALITY
     type: string
+  - name: VSCODE_RUN_UNIT_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_INTEGRATION_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_SMOKE_TESTS
+    type: boolean
+    default: true
 
 steps:
   - task: NodeTool@0
@@ -165,91 +174,126 @@ steps:
         VSCODE_ARCH=$(VSCODE_ARCH) DEBUG=electron-osx-sign* node build/darwin/sign.js
       displayName: Set Hardened Entitlements
 
-  - script: |
-      set -e
-      ./scripts/test.sh --build --tfs "Unit Tests"
-    displayName: Run unit tests (Electron)
-    timeoutInMinutes: 15
-
-  - script: |
-      set -e
-      yarn test-node --build
-    displayName: Run unit tests (node.js)
-    timeoutInMinutes: 15
-
-  - script: |
-      set -e
-      DEBUG=*browser* yarn test-browser-no-install --sequential --build --browser chromium --browser webkit --tfs "Browser Unit Tests"
-    displayName: Run unit tests (Browser, Chromium & Webkit)
-    timeoutInMinutes: 30
-
-  - script: |
-      # Figure out the full absolute path of the product we just built
-      # including the remote server and configure the integration tests
-      # to run with these builds instead of running out of sources.
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
-      ./scripts/test-integration.sh --build --tfs "Integration Tests"
-    displayName: Run integration tests (Electron)
-    timeoutInMinutes: 20
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        ./scripts/test.sh --build --tfs "Unit Tests"
+      displayName: Run unit tests (Electron)
+      timeoutInMinutes: 15
 
-  - script: |
-      set -e
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
-      ./scripts/test-web-integration.sh --browser webkit
-    displayName: Run integration tests (Browser, Webkit)
-    timeoutInMinutes: 20
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn test-node --build
+      displayName: Run unit tests (node.js)
+      timeoutInMinutes: 15
 
-  - script: |
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
-      ./scripts/test-remote-integration.sh
-    displayName: Run integration tests (Remote)
-    timeoutInMinutes: 20
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        DEBUG=*browser* yarn test-browser-no-install --sequential --build --browser chromium --browser webkit --tfs "Browser Unit Tests"
+      displayName: Run unit tests (Browser, Chromium & Webkit)
+      timeoutInMinutes: 30
 
-  - script: |
-      set -e
-      ps -ef
-    displayName: Diagnostics before smoke test run
-    continueOnError: true
-    condition: succeededOrFailed()
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn gulp \
+          compile-extension:css-language-features-server \
+          compile-extension:emmet \
+          compile-extension:git \
+          compile-extension:github-authentication \
+          compile-extension:html-language-features-server \
+          compile-extension:ipynb \
+          compile-extension:json-language-features-server \
+          compile-extension:markdown-language-features \
+          compile-extension-media \
+          compile-extension:microsoft-authentication \
+          compile-extension:typescript-language-features \
+          compile-extension:vscode-api-tests \
+          compile-extension:vscode-colorize-tests \
+          compile-extension:vscode-custom-editor-tests \
+          compile-extension:vscode-notebook-tests \
+          compile-extension:vscode-test-resolver
+      displayName: Build integration tests
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        # Figure out the full absolute path of the product we just built
+        # including the remote server and configure the integration tests
+        # to run with these builds instead of running out of sources.
+        set -e
+        APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+        APP_NAME="`ls $APP_ROOT | head -n 1`"
+        INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
+        ./scripts/test-integration.sh --build --tfs "Integration Tests"
+      displayName: Run integration tests (Electron)
+      timeoutInMinutes: 20
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
+        ./scripts/test-web-integration.sh --browser webkit
+      displayName: Run integration tests (Browser, Webkit)
+      timeoutInMinutes: 20
 
-  - script: |
-      set -e
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
-      yarn smoketest-no-compile --web --tracing --headless
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Browser, Chromium)
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+        APP_NAME="`ls $APP_ROOT | head -n 1`"
+        INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME/Contents/MacOS/Electron" \
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
+        ./scripts/test-remote-integration.sh
+      displayName: Run integration tests (Remote)
+      timeoutInMinutes: 20
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        ps -ef
+      displayName: Diagnostics before smoke test run
+      continueOnError: true
+      condition: succeededOrFailed()
 
-  - script: |
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      yarn smoketest-no-compile --tracing --build "$APP_ROOT/$APP_NAME"
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Electron)
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
+        yarn smoketest-no-compile --web --tracing --headless
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Browser, Chromium)
 
-  - script: |
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
-      APP_NAME="`ls $APP_ROOT | head -n 1`"
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
-      yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Remote)
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+        APP_NAME="`ls $APP_ROOT | head -n 1`"
+        yarn smoketest-no-compile --tracing --build "$APP_ROOT/$APP_NAME"
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Electron)
 
-  - script: |
-      set -e
-      ps -ef
-    displayName: Diagnostics after smoke test run
-    continueOnError: true
-    condition: succeededOrFailed()
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn gulp compile-extension:vscode-test-resolver
+        APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+        APP_NAME="`ls $APP_ROOT | head -n 1`"
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
+        yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Remote)
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        ps -ef
+      displayName: Diagnostics after smoke test run
+      continueOnError: true
+      condition: succeededOrFailed()
 
   - task: PublishPipelineArtifact@0
     inputs:
diff --git a/build/azure-pipelines/linux/product-build-linux-client.yml b/build/azure-pipelines/linux/product-build-linux-client.yml
index 4b82e42689c..4e6f8f13d0e 100644
--- a/build/azure-pipelines/linux/product-build-linux-client.yml
+++ b/build/azure-pipelines/linux/product-build-linux-client.yml
@@ -1,6 +1,15 @@
 parameters:
   - name: VSCODE_QUALITY
     type: string
+  - name: VSCODE_RUN_UNIT_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_INTEGRATION_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_SMOKE_TESTS
+    type: boolean
+    default: true
 
 steps:
   - task: NodeTool@0
@@ -221,104 +230,139 @@ steps:
       stat $ELECTRON_ROOT/chrome-sandbox
     displayName: Change setuid helper binary permission
 
-  - script: |
-      set -e
-      ./scripts/test.sh --build --tfs "Unit Tests"
-    displayName: Run unit tests (Electron)
-    timeoutInMinutes: 15
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
-  - script: |
-      set -e
-      yarn test-node --build
-    displayName: Run unit tests (node.js)
-    timeoutInMinutes: 15
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
-  - script: |
-      set -e
-      DEBUG=*browser* yarn test-browser-no-install --build --browser chromium --tfs "Browser Unit Tests"
-    displayName: Run unit tests (Browser, Chromium)
-    timeoutInMinutes: 15
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
-  - script: |
-      # Figure out the full absolute path of the product we just built
-      # including the remote server and configure the integration tests
-      # to run with these builds instead of running out of sources.
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
-      APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
-      INTEGRATION_TEST_APP_NAME="$APP_NAME" \
-      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
-      ./scripts/test-integration.sh --build --tfs "Integration Tests"
-    displayName: Run integration tests (Electron)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
-
-  - script: |
-      set -e
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
-      ./scripts/test-web-integration.sh --browser chromium
-    displayName: Run integration tests (Browser, Chromium)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        ./scripts/test.sh --build --tfs "Unit Tests"
+      displayName: Run unit tests (Electron)
+      timeoutInMinutes: 15
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
-      APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
-      INTEGRATION_TEST_APP_NAME="$APP_NAME" \
-      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
-      ./scripts/test-remote-integration.sh
-    displayName: Run integration tests (Remote)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn test-node --build
+      displayName: Run unit tests (node.js)
+      timeoutInMinutes: 15
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      ps -ef
-      cat /proc/sys/fs/inotify/max_user_watches
-      lsof | wc -l
-    displayName: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles)
-    continueOnError: true
-    condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - script: |
+        set -e
+        DEBUG=*browser* yarn test-browser-no-install --build --browser chromium --tfs "Browser Unit Tests"
+      displayName: Run unit tests (Browser, Chromium)
+      timeoutInMinutes: 15
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
-      yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Browser, Chromium)
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn gulp \
+          compile-extension:css-language-features-server \
+          compile-extension:emmet \
+          compile-extension:git \
+          compile-extension:github-authentication \
+          compile-extension:html-language-features-server \
+          compile-extension:ipynb \
+          compile-extension:json-language-features-server \
+          compile-extension:markdown-language-features \
+          compile-extension-media \
+          compile-extension:microsoft-authentication \
+          compile-extension:typescript-language-features \
+          compile-extension:vscode-api-tests \
+          compile-extension:vscode-colorize-tests \
+          compile-extension:vscode-custom-editor-tests \
+          compile-extension:vscode-notebook-tests \
+          compile-extension:vscode-test-resolver
+      displayName: Build integration tests
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        # Figure out the full absolute path of the product we just built
+        # including the remote server and configure the integration tests
+        # to run with these builds instead of running out of sources.
+        set -e
+        APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+        APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
+        INTEGRATION_TEST_APP_NAME="$APP_NAME" \
+        INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
+        ./scripts/test-integration.sh --build --tfs "Integration Tests"
+      displayName: Run integration tests (Electron)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
+        ./scripts/test-web-integration.sh --browser chromium
+      displayName: Run integration tests (Browser, Chromium)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
-      yarn smoketest-no-compile --tracing --build "$APP_PATH"
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Electron)
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - script: |
+        set -e
+        APP_ROOT=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+        APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
+        INTEGRATION_TEST_APP_NAME="$APP_NAME" \
+        INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
+        ./scripts/test-remote-integration.sh
+      displayName: Run integration tests (Remote)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        ps -ef
+        cat /proc/sys/fs/inotify/max_user_watches
+        lsof | wc -l
+      displayName: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles)
+      continueOnError: true
+      condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
+        yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Browser, Chromium)
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
-      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
-      yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
-    timeoutInMinutes: 20
-    displayName: Run smoke tests (Remote)
-    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+        yarn smoketest-no-compile --tracing --build "$APP_PATH"
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Electron)
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - script: |
-      set -e
-      ps -ef
-      cat /proc/sys/fs/inotify/max_user_watches
-      lsof | wc -l
-    displayName: Diagnostics after smoke test run (processes, max_user_watches, number of opened file handles)
-    continueOnError: true
-    condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        yarn gulp compile-extension:vscode-test-resolver
+        APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+        VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
+        yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
+      timeoutInMinutes: 20
+      displayName: Run smoke tests (Remote)
+      condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - script: |
+        set -e
+        ps -ef
+        cat /proc/sys/fs/inotify/max_user_watches
+        lsof | wc -l
+      displayName: Diagnostics after smoke test run (processes, max_user_watches, number of opened file handles)
+      continueOnError: true
+      condition: and(succeededOrFailed(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0
     inputs:
diff --git a/build/azure-pipelines/product-build-pr.yml b/build/azure-pipelines/product-build-pr.yml
index 21fbe295cc9..5b09f8ee77e 100644
--- a/build/azure-pipelines/product-build-pr.yml
+++ b/build/azure-pipelines/product-build-pr.yml
@@ -60,7 +60,8 @@ stages:
       - Compile
     pool: vscode-1es-vscode-windows-2019
     jobs:
-      - job: Windows
+      - job: WindowsUnitTests
+        displayName: Unit Tests
         timeoutInMinutes: 120
         variables:
           VSCODE_ARCH: x64
@@ -68,6 +69,33 @@ stages:
           - template: win32/product-build-win32.yml
             parameters:
               VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: true
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: WindowsIntegrationTests
+        displayName: Integration Tests
+        timeoutInMinutes: 120
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: win32/product-build-win32.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: true
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: WindowsSmokeTests
+        displayName: Smoke Tests
+        timeoutInMinutes: 120
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: win32/product-build-win32.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: true
 
   - stage: Linux
     dependsOn:
@@ -75,7 +103,22 @@ stages:
       - LinuxServerDependencies
     pool: vscode-1es-vscode-linux-18.04
     jobs:
-      - job: Linuxx64
+      - job: Linuxx64UnitTest
+        displayName: Unit Tests
+        container: vscode-bionic-x64
+        variables:
+          VSCODE_ARCH: x64
+          NPM_ARCH: x64
+          DISPLAY: ":10"
+        steps:
+          - template: linux/product-build-linux-client.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: true
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: Linuxx64IntegrationTest
+        displayName: Integration Tests
         container: vscode-bionic-x64
         variables:
           VSCODE_ARCH: x64
@@ -85,6 +128,23 @@ stages:
           - template: linux/product-build-linux-client.yml
             parameters:
               VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: true
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: Linuxx64SmokeTest
+        displayName: Smoke Tests
+        container: vscode-bionic-x64
+        variables:
+          VSCODE_ARCH: x64
+          NPM_ARCH: x64
+          DISPLAY: ":10"
+        steps:
+          - template: linux/product-build-linux-client.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: true
 
   - stage: macOS
     dependsOn:
@@ -94,7 +154,32 @@ stages:
     variables:
       BUILDSECMON_OPT_IN: true
     jobs:
-      - job: macOSTest
+      - job: macOSUnitTest
+        displayName: Unit Tests
+        timeoutInMinutes: 90
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: darwin/product-build-darwin-test.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: true
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: macOSIntegrationTest
+        displayName: Integration Tests
+        timeoutInMinutes: 90
+        variables:
+          VSCODE_ARCH: x64
+        steps:
+          - template: darwin/product-build-darwin-test.yml
+            parameters:
+              VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: true
+              VSCODE_RUN_SMOKE_TESTS: false
+      - job: macOSSmokeTest
+        displayName: Smoke Tests
         timeoutInMinutes: 90
         variables:
           VSCODE_ARCH: x64
@@ -102,3 +187,6 @@ stages:
           - template: darwin/product-build-darwin-test.yml
             parameters:
               VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }}
+              VSCODE_RUN_UNIT_TESTS: false
+              VSCODE_RUN_INTEGRATION_TESTS: false
+              VSCODE_RUN_SMOKE_TESTS: true
diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml
index d089f70d228..286283a31dc 100644
--- a/build/azure-pipelines/win32/product-build-win32.yml
+++ b/build/azure-pipelines/win32/product-build-win32.yml
@@ -1,6 +1,15 @@
 parameters:
   - name: VSCODE_QUALITY
     type: string
+  - name: VSCODE_RUN_UNIT_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_INTEGRATION_TESTS
+    type: boolean
+    default: true
+  - name: VSCODE_RUN_SMOKE_TESTS
+    type: boolean
+    default: true
 
 steps:
   - task: NodeTool@0
@@ -184,105 +193,142 @@ steps:
     displayName: Download Playwright
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      exec { yarn electron $(VSCODE_ARCH) }
-      exec { .\scripts\test.bat --build --tfs "Unit Tests" }
-    displayName: Run unit tests (Electron)
-    timeoutInMinutes: 15
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
-
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      exec { yarn test-node --build }
-    displayName: Run unit tests (node.js)
-    timeoutInMinutes: 15
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
-
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      exec { yarn test-browser-no-install --sequential --build --browser chromium --browser firefox --tfs "Browser Unit Tests" }
-    displayName: Run unit tests (Browser, Chromium & Firefox)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
-
-  - powershell: |
-      # Figure out the full absolute path of the product we just built
-      # including the remote server and configure the integration tests
-      # to run with these builds instead of running out of sources.
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
-      $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
-      $AppNameShort = $AppProductJson.nameShort
-      exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-integration.bat --build --tfs "Integration Tests" }
-    displayName: Run integration tests (Electron)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        exec { yarn electron $(VSCODE_ARCH) }
+        exec { .\scripts\test.bat --build --tfs "Unit Tests" }
+      displayName: Run unit tests (Electron)
+      timeoutInMinutes: 15
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      exec { $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"; .\scripts\test-web-integration.bat --browser firefox }
-    displayName: Run integration tests (Browser, Firefox)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        exec { yarn test-node --build }
+      displayName: Run unit tests (node.js)
+      timeoutInMinutes: 15
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
-      $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
-      $AppNameShort = $AppProductJson.nameShort
-      exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-remote-integration.bat }
-    displayName: Run integration tests (Remote)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_UNIT_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        exec { yarn test-browser-no-install --sequential --build --browser chromium --browser firefox --tfs "Browser Unit Tests" }
+      displayName: Run unit tests (Browser, Chromium & Firefox)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      exec {.\build\azure-pipelines\win32\listprocesses.bat }
-    displayName: Diagnostics before smoke test run
-    continueOnError: true
-    condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        exec { yarn gulp `
+          compile-extension:css-language-features-server `
+          compile-extension:emmet `
+          compile-extension:git `
+          compile-extension:github-authentication `
+          compile-extension:html-language-features-server `
+          compile-extension:ipynb `
+          compile-extension:json-language-features-server `
+          compile-extension:markdown-language-features `
+          compile-extension-media `
+          compile-extension:microsoft-authentication `
+          compile-extension:typescript-language-features `
+          compile-extension:vscode-api-tests `
+          compile-extension:vscode-colorize-tests `
+          compile-extension:vscode-custom-editor-tests `
+          compile-extension:vscode-notebook-tests `
+          compile-extension:vscode-test-resolver `
+        }
+      displayName: Build integration tests
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - powershell: |
+        # Figure out the full absolute path of the product we just built
+        # including the remote server and configure the integration tests
+        # to run with these builds instead of running out of sources.
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+        $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
+        $AppNameShort = $AppProductJson.nameShort
+        exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-integration.bat --build --tfs "Integration Tests" }
+      displayName: Run integration tests (Electron)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        exec { $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"; .\scripts\test-web-integration.bat --browser firefox }
+      displayName: Run integration tests (Browser, Firefox)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
-      exec { yarn smoketest-no-compile --web --tracing --headless }
-    displayName: Run smoke tests (Browser, Chromium)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_INTEGRATION_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+        $AppProductJson = Get-Content -Raw -Path "$AppRoot\resources\app\product.json" | ConvertFrom-Json
+        $AppNameShort = $AppProductJson.nameShort
+        exec { $env:INTEGRATION_TEST_ELECTRON_PATH = "$AppRoot\$AppNameShort.exe"; $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"; .\scripts\test-remote-integration.bat }
+      displayName: Run integration tests (Remote)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        exec {.\build\azure-pipelines\win32\listprocesses.bat }
+      displayName: Diagnostics before smoke test run
+      continueOnError: true
+      condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
-      exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
-    displayName: Run smoke tests (Electron)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
+        exec { yarn smoketest-no-compile --web --tracing --headless }
+      displayName: Run smoke tests (Browser, Chromium)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
-      $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
-      exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
-    displayName: Run smoke tests (Remote)
-    timeoutInMinutes: 20
-    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+        exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
+      displayName: Run smoke tests (Electron)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      exec {.\build\azure-pipelines\win32\listprocesses.bat }
-    displayName: Diagnostics after smoke test run
-    continueOnError: true
-    condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        $ErrorActionPreference = "Stop"
+        $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+        $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
+        exec { yarn gulp compile-extension:vscode-test-resolver }
+        exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
+      displayName: Run smoke tests (Remote)
+      timeoutInMinutes: 20
+      condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - ${{ if eq(parameters.VSCODE_RUN_SMOKE_TESTS, true) }}:
+    - powershell: |
+        . build/azure-pipelines/win32/exec.ps1
+        exec {.\build\azure-pipelines\win32\listprocesses.bat }
+      displayName: Diagnostics after smoke test run
+      continueOnError: true
+      condition: and(succeededOrFailed(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0
     inputs: