|
| 1 | +// Copyright (c) Microsoft Corporation. All rights reserved. |
| 2 | +// Licensed under the MIT License. |
| 3 | + |
| 4 | +'use strict'; |
| 5 | + |
| 6 | +/* |
| 7 | +Comparing performance metrics is not easy (the metrics can and always get skewed). |
| 8 | +One approach is to run the tests multile times and gather multiple sample data. |
| 9 | +For Extension activation times, we load both extensions x times, and re-load the window y times in each x load. |
| 10 | +I.e. capture averages by giving the extensions sufficient time to warm up. |
| 11 | +This block of code merely launches the tests by using either the dev or release version of the extension, |
| 12 | +and spawning the tests (mimic user starting tests from command line), this way we can run tests multiple times. |
| 13 | +*/ |
| 14 | + |
| 15 | +// tslint:disable:no-console no-require-imports no-var-requires |
| 16 | + |
| 17 | +import { spawn } from 'child_process'; |
| 18 | +import * as download from 'download'; |
| 19 | +import * as fs from 'fs-extra'; |
| 20 | +import * as path from 'path'; |
| 21 | +import * as request from 'request'; |
| 22 | +import { EXTENSION_ROOT_DIR } from '../client/common/constants'; |
| 23 | + |
| 24 | +const NamedRegexp = require('named-js-regexp'); |
| 25 | +const StreamZip = require('node-stream-zip'); |
| 26 | +const del = require('del'); |
| 27 | + |
| 28 | +const tmpFolder = path.join(EXTENSION_ROOT_DIR, 'tmp'); |
| 29 | +const publishedExtensionPath = path.join(tmpFolder, 'ext', 'testReleaseExtensionsFolder'); |
| 30 | +const logFilesPath = path.join(tmpFolder, 'test', 'logs'); |
| 31 | + |
| 32 | +enum Version { |
| 33 | + Dev, Release |
| 34 | +} |
| 35 | + |
| 36 | +class TestRunner { |
| 37 | + public async start() { |
| 38 | + await del([path.join(tmpFolder, '**')]); |
| 39 | + await this.extractLatestExtension(publishedExtensionPath); |
| 40 | + |
| 41 | + const timesToLoadEachVersion = 3; |
| 42 | + const devLogFiles: string[] = []; |
| 43 | + const releaseLogFiles: string[] = []; |
| 44 | + const newAnalysisEngineLogFiles: string[] = []; |
| 45 | + |
| 46 | + for (let i = 0; i < timesToLoadEachVersion; i += 1) { |
| 47 | + await this.enableNewAnalysisEngine(false); |
| 48 | + |
| 49 | + const devLogFile = path.join(logFilesPath, `dev_loadtimes${i}.txt`); |
| 50 | + await this.capturePerfTimes(Version.Dev, devLogFile); |
| 51 | + devLogFiles.push(devLogFile); |
| 52 | + |
| 53 | + const releaseLogFile = path.join(logFilesPath, `release_loadtimes${i}.txt`); |
| 54 | + await this.capturePerfTimes(Version.Release, releaseLogFile); |
| 55 | + releaseLogFiles.push(releaseLogFile); |
| 56 | + |
| 57 | + // New Analysis engine. |
| 58 | + await this.enableNewAnalysisEngine(true); |
| 59 | + const newAnalysisEngineLogFile = path.join(logFilesPath, `newAnalysisEngine_loadtimes${i}.txt`); |
| 60 | + await this.capturePerfTimes(Version.Release, newAnalysisEngineLogFile); |
| 61 | + newAnalysisEngineLogFiles.push(newAnalysisEngineLogFile); |
| 62 | + } |
| 63 | + |
| 64 | + await this.runPerfTest(devLogFiles, releaseLogFiles, newAnalysisEngineLogFiles); |
| 65 | + } |
| 66 | + private async enableNewAnalysisEngine(enable: boolean) { |
| 67 | + const settings = `{ "python.jediEnabled": ${!enable} }`; |
| 68 | + await fs.writeFile(path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'performance', 'settings.json'), settings); |
| 69 | + } |
| 70 | + |
| 71 | + private async capturePerfTimes(version: Version, logFile: string) { |
| 72 | + const releaseVersion = await this.getReleaseVersion(); |
| 73 | + const devVersion = await this.getDevVersion(); |
| 74 | + await fs.ensureDir(path.dirname(logFile)); |
| 75 | + const env: { [key: string]: {} } = { |
| 76 | + ACTIVATION_TIMES_LOG_FILE_PATH: logFile, |
| 77 | + ACTIVATION_TIMES_EXT_VERSION: version === Version.Release ? releaseVersion : devVersion, |
| 78 | + CODE_EXTENSIONS_PATH: version === Version.Release ? publishedExtensionPath : EXTENSION_ROOT_DIR |
| 79 | + }; |
| 80 | + |
| 81 | + await this.launchTest(env); |
| 82 | + } |
| 83 | + private async runPerfTest(devLogFiles: string[], releaseLogFiles: string[], newAnalysisEngineLogFiles: string[]) { |
| 84 | + const env: { [key: string]: {} } = { |
| 85 | + ACTIVATION_TIMES_DEV_LOG_FILE_PATHS: JSON.stringify(devLogFiles), |
| 86 | + ACTIVATION_TIMES_RELEASE_LOG_FILE_PATHS: JSON.stringify(releaseLogFiles), |
| 87 | + ACTIVATION_TIMES_DEV_ANALYSIS_LOG_FILE_PATHS: JSON.stringify(newAnalysisEngineLogFiles) |
| 88 | + }; |
| 89 | + |
| 90 | + await this.launchTest(env); |
| 91 | + } |
| 92 | + |
| 93 | + private async launchTest(customEnvVars: { [key: string]: {} }) { |
| 94 | + await new Promise((resolve, reject) => { |
| 95 | + const env: { [key: string]: {} } = { |
| 96 | + TEST_FILES_SUFFIX: 'perf.test', |
| 97 | + CODE_TESTS_WORKSPACE: path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'performance'), |
| 98 | + ...process.env, |
| 99 | + ...customEnvVars |
| 100 | + }; |
| 101 | + |
| 102 | + const proc = spawn('node', [path.join(__dirname, 'standardTest.js')], { cwd: EXTENSION_ROOT_DIR, env }); |
| 103 | + proc.stdout.pipe(process.stdout); |
| 104 | + proc.stderr.pipe(process.stderr); |
| 105 | + proc.on('error', reject); |
| 106 | + proc.on('close', code => { |
| 107 | + if (code === 0) { |
| 108 | + resolve(); |
| 109 | + } else { |
| 110 | + reject(`Failed with code ${code}.`); |
| 111 | + } |
| 112 | + }); |
| 113 | + }); |
| 114 | + } |
| 115 | + |
| 116 | + private async extractLatestExtension(targetDir: string): Promise<void> { |
| 117 | + const extensionFile = await this.downloadExtension(); |
| 118 | + await this.unzip(extensionFile, targetDir); |
| 119 | + } |
| 120 | + |
| 121 | + private async getReleaseVersion(): Promise<string> { |
| 122 | + const url = 'https://marketplace.visualstudio.com/items?itemName=ms-python.python'; |
| 123 | + const content = await new Promise<string>((resolve, reject) => { |
| 124 | + request(url, (error, response, body) => { |
| 125 | + if (error) { |
| 126 | + return reject(error); |
| 127 | + } |
| 128 | + if (response.statusCode === 200) { |
| 129 | + return resolve(body); |
| 130 | + } |
| 131 | + reject(`Status code of ${response.statusCode} received.`); |
| 132 | + }); |
| 133 | + }); |
| 134 | + const re = NamedRegexp('"version"\S?:\S?"(:<version>\\d{4}\\.\\d{1,2}\\.\\d{1,2})"', 'g'); |
| 135 | + const matches = re.exec(content); |
| 136 | + return matches.groups().version; |
| 137 | + } |
| 138 | + |
| 139 | + private async getDevVersion(): Promise<string> { |
| 140 | + // tslint:disable-next-line:non-literal-require |
| 141 | + return require(path.join(EXTENSION_ROOT_DIR, 'package.json')).version; |
| 142 | + } |
| 143 | + |
| 144 | + private async unzip(zipFile: string, targetFolder: string): Promise<void> { |
| 145 | + await fs.ensureDir(targetFolder); |
| 146 | + return new Promise<void>((resolve, reject) => { |
| 147 | + const zip = new StreamZip({ |
| 148 | + file: zipFile, |
| 149 | + storeEntries: true |
| 150 | + }); |
| 151 | + zip.on('ready', async () => { |
| 152 | + zip.extract('extension', targetFolder, err => { |
| 153 | + if (err) { |
| 154 | + reject(err); |
| 155 | + } else { |
| 156 | + resolve(); |
| 157 | + } |
| 158 | + zip.close(); |
| 159 | + }); |
| 160 | + }); |
| 161 | + }); |
| 162 | + } |
| 163 | + |
| 164 | + private async downloadExtension(): Promise<string> { |
| 165 | + const version = await this.getReleaseVersion(); |
| 166 | + const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/ms-python/vsextensions/python/${version}/vspackage`; |
| 167 | + const destination = path.join(__dirname, `extension${version}.zip`); |
| 168 | + if (await fs.pathExists(destination)) { |
| 169 | + return destination; |
| 170 | + } |
| 171 | + |
| 172 | + await download(url, path.dirname(destination), { filename: path.basename(destination) }); |
| 173 | + return destination; |
| 174 | + } |
| 175 | +} |
| 176 | + |
| 177 | +new TestRunner().start().catch(ex => console.error('Error in running Performance Tests', ex)); |
0 commit comments