1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { Suite, Context } from 'mocha';
import { dirname, join } from 'path';
import { Application, ApplicationOptions, Logger } from '../../automation';
export function describeRepeat(n: number, description: string, callback: (this: Suite) => void): void {
for (let i = 0; i < n; i++) {
describe(`${description} (iteration ${i})`, callback);
}
}
export function itRepeat(n: number, description: string, callback: (this: Context) => any): void {
for (let i = 0; i < n; i++) {
it(`${description} (iteration ${i})`, callback);
}
}
/**
* Defines a test-case that will run but will be skips it if it throws an exception. This is useful
* to get some runs in CI when trying to stabilize a flaky test, without failing the build. Note
* that this only works if something inside the test throws, so a test's overall timeout won't work
* but throwing due to a polling timeout will.
* @param title The test-case title.
* @param callback The test-case callback.
*/
export function itSkipOnFail(title: string, callback: (this: Context) => any): void {
it(title, function () {
return Promise.resolve().then(() => {
return callback.apply(this, arguments);
}).catch(e => {
console.warn(`Test "${title}" failed but was marked as skip on fail:`, e);
this.skip();
});
});
}
export function installAllHandlers(logger: Logger, optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) {
installDiagnosticsHandler(logger);
installAppBeforeHandler(optionsTransform);
installAppAfterHandler();
}
export function installDiagnosticsHandler(logger: Logger, appFn?: () => Application | undefined) {
// Before each suite
before(async function () {
const suiteTitle = this.currentTest?.parent?.title;
logger.log('');
logger.log(`>>> Suite start: '${suiteTitle ?? 'unknown'}' <<<`);
logger.log('');
});
// Before each test
beforeEach(async function () {
const testTitle = this.currentTest?.title;
logger.log('');
logger.log(`>>> Test start: '${testTitle ?? 'unknown'}' <<<`);
logger.log('');
const app: Application = appFn?.() ?? this.app;
await app?.startTracing(testTitle ?? 'unknown');
});
// After each test
afterEach(async function () {
const currentTest = this.currentTest;
if (!currentTest) {
return;
}
const failed = currentTest.state === 'failed';
const testTitle = currentTest.title;
logger.log('');
if (failed) {
logger.log(`>>> !!! FAILURE !!! Test end: '${testTitle}' !!! FAILURE !!! <<<`);
} else {
logger.log(`>>> Test end: '${testTitle}' <<<`);
}
logger.log('');
const app: Application = appFn?.() ?? this.app;
await app?.stopTracing(testTitle.replace(/[^a-z0-9\-]/ig, '_'), failed);
});
}
let logsCounter = 1;
export function suiteLogsPath(options: ApplicationOptions, suiteName: string): string {
return join(dirname(options.logsPath), `${logsCounter++}_suite_${suiteName.replace(/[^a-z0-9\-]/ig, '_')}`);
}
function installAppBeforeHandler(optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) {
before(async function () {
const suiteName = this.test?.parent?.title ?? 'unknown';
this.app = createApp({
...this.defaultOptions,
logsPath: suiteLogsPath(this.defaultOptions, suiteName)
}, optionsTransform);
await this.app.start();
});
}
export function installAppAfterHandler(appFn?: () => Application | undefined, joinFn?: () => Promise<unknown>) {
after(async function () {
const app: Application = appFn?.() ?? this.app;
if (app) {
await app.stop();
}
if (joinFn) {
await joinFn();
}
});
}
export function createApp(options: ApplicationOptions, optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions): Application {
if (optionsTransform) {
options = optionsTransform({ ...options });
}
const app = new Application({
...options,
userDataDir: getRandomUserDataDir(options)
});
return app;
}
export function getRandomUserDataDir(options: ApplicationOptions): string {
// Pick a random user data dir suffix that is not
// too long to not run into max path length issues
// https://github.com/microsoft/vscode/issues/34988
const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join('');
return options.userDataDir.concat(`-${userDataPathSuffix}`);
}
export function timeout(i: number) {
return new Promise<void>(resolve => {
setTimeout(() => {
resolve();
}, i);
});
}
export async function retryWithRestart(app: Application, testFn: () => Promise<unknown>, retries = 3, timeoutMs = 20000): Promise<unknown> {
let lastError: Error | undefined = undefined;
for (let i = 0; i < retries; i++) {
const result = await Promise.race([
testFn().then(() => true, error => {
lastError = error;
return false;
}),
timeout(timeoutMs).then(() => false)
]);
if (result) {
return;
}
await app.restart();
}
throw lastError ?? new Error('retryWithRestart failed with an unknown error');
}
export interface ITask<T> {
(): T;
}
export async function retry<T>(task: ITask<Promise<T>>, delay: number, retries: number, onBeforeRetry?: () => Promise<unknown>): Promise<T> {
let lastError: Error | undefined;
for (let i = 0; i < retries; i++) {
try {
if (i > 0 && typeof onBeforeRetry === 'function') {
try {
await onBeforeRetry();
} catch (error) {
console.warn(`onBeforeRetry failed with: ${error}`);
}
}
return await task();
} catch (error) {
lastError = error;
await timeout(delay);
}
}
throw lastError;
}
|
