smoke - run legacy tests again, do not fail new playwright tests for a while to figure out flakiness

3 files changed, 64 insertions, 0 deletions

diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml
index 1c529a95362..2b4a19c42aa 100644
--- a/build/azure-pipelines/darwin/product-build-darwin.yml
+++ b/build/azure-pipelines/darwin/product-build-darwin.yml
@@ -249,6 +249,7 @@ steps:
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - script: |
@@ -259,6 +260,27 @@ steps:
       yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME"
     timeoutInMinutes: 10
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      yarn smoketest-no-compile --legacy --tracing --build "$APP_ROOT/$APP_NAME"
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    displayName: Run smoke tests (Legacy, Electron)
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
+      APP_NAME="`ls $APP_ROOT | head -n 1`"
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \
+      yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_ROOT/$APP_NAME"
+    timeoutInMinutes: 10
+    displayName: Run smoke tests (Legacy, Remote)
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0
diff --git a/build/azure-pipelines/linux/product-build-linux-client.yml b/build/azure-pipelines/linux/product-build-linux-client.yml
index 5f15732ad85..3e04ef34233 100644
--- a/build/azure-pipelines/linux/product-build-linux-client.yml
+++ b/build/azure-pipelines/linux/product-build-linux-client.yml
@@ -271,6 +271,7 @@ steps:
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - script: |
@@ -280,6 +281,25 @@ steps:
       yarn smoketest-no-compile --tracing --remote --build "$APP_PATH"
     timeoutInMinutes: 10
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+      yarn smoketest-no-compile --legacy --tracing --build "$APP_PATH"
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    displayName: Run smoke tests (Legacy, Electron)
+    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
+
+  - script: |
+      set -e
+      APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
+      yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_PATH"
+    timeoutInMinutes: 10
+    displayName: Run smoke tests (Legacy, Remote)
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - task: PublishPipelineArtifact@0
diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml
index cd18ffe4c7a..b68783b2e2c 100644
--- a/build/azure-pipelines/win32/product-build-win32.yml
+++ b/build/azure-pipelines/win32/product-build-win32.yml
@@ -231,6 +231,7 @@ steps:
       $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
       exec { yarn smoketest-no-compile --tracing --build "$AppRoot" }
     displayName: Run smoke tests (Electron)
+    continueOnError: true
     # Increased timeout because this test downloads stable code
     timeoutInMinutes: 20
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
@@ -242,6 +243,27 @@ steps:
       $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
       exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" }
     displayName: Run smoke tests (Remote)
+    continueOnError: true
+    timeoutInMinutes: 10
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --legacy --tracing --build "$AppRoot" }
+    displayName: Run smoke tests (Legacy, Electron)
+    # Increased timeout because this test downloads stable code
+    timeoutInMinutes: 20
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
+
+  - powershell: |
+      . build/azure-pipelines/win32/exec.ps1
+      $ErrorActionPreference = "Stop"
+      $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
+      $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
+      exec { yarn smoketest-no-compile --legacy --tracing --remote --build "$AppRoot" }
+    displayName: Run smoke tests (Legacy, Remote)
     timeoutInMinutes: 10
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))