Fix functional tests for nightly run (microsoft#4093)

* Fix functional tests on linux * Fix problems on windows * Add news entry * Don't send envVars when not set. * Fix hygiene and back out pythonExecutionFactory change * Try fixing the functional requirements * Make a different test phase for just 3.4 so functional tests are skipped

Author: rchiodo

build/.mocha.functional.opts

@@ -1,7 +1,7 @@
 ./out/test/**/*.functional.test.js
---require out/test/unittests.js
---exclude out/**/*.jsx
---ui tdd
+--require=out/test/unittests.js
+--exclude=out/**/*.jsx
+--ui=tdd
 --recursive
 --colors
---timeout 120000
+--timeout=120000

build/ci/templates/test-phase-job-3-4.yml

@@ -0,0 +1,47 @@
+parameters:
+ PythonVersion: '3.7'
+ NodeVersion: '8.11.2'
+ NpmVersion: 'latest'
+ PoolName: 'Hosted Ubuntu 1604'
+ MOCHA_CI_REPORTER_ID: '$(Build.SourcesDirectory)/build/ci/mocha-vsts-reporter.js'
+ MOCHA_CI_REPORTFILE: '$(Build.ArtifactStagingDirectory)/reports/junit-report.xml'
+ MOCHA_REPORTER_JUNIT: true
+ UploadBinary: false
+ AzureStorageAccountName: 'vscodepythonci'
+ AzureStorageContainerName: 'vscode-python-ci'
+ BuildNumber: '$(Build.BuildNumber)'
+ Platform: 'Linux'
+ DependsOn: 'Prebuild'
+
+jobs:
+- job:
+ displayName: ${{ format('SystemTest {0} Py{1}', parameters.Platform, parameters.PythonVersion) }}
+ dependsOn: ${{ parameters.DependsOn }}
+ pool: ${{ parameters.pool }}
+
+ variables:
+ # TODO: use {{ insert }}: {{ parameters.variables }}, it would not work at time I wrote this
+ nodeVersion: ${{ parameters.NodeVersion }}
+ npmVersion: ${{ parameters.NpmVersion }}
+ pythonVersion: ${{ parameters.PythonVersion }}
+ mochaReportFile: ${{ parameters.MOCHA_CI_REPORTFILE }}
+ MOCHA_CI_REPORTER_ID: ${{ parameters.MOCHA_CI_REPORTER_ID }}
+ MOCHA_CI_REPORTFILE: ${{ parameters.MOCHA_CI_REPORTFILE }}
+ MOCHA_REPORTER_JUNIT: ${{ parameters.MOCHA_REPORTER_JUNIT }}
+ uploadBinary: ${{ parameters.UploadBinary }}
+ azureStorageAccountName: ${{ parameters.AzureStorageAccountName }}
+ azureStorageContainerName: ${{ parameters.AzureStorageContainerName }}
+ platform: ${{ parameters.Platform }}
+ buildNumber: ${{ parameters.BuildNumber }}
+
+ strategy:
+ matrix:
+ SingleWorkspace:
+ TestSuiteName: 'testSingleWorkspace'
+ MultiWorkspace:
+ TestSuiteName: 'testMultiWorkspace'
+ Debugger:
+ TestSuiteName: 'testDebugger'
+
+ steps:
+ - template: test-phase.yml

build/ci/vscode-python-nightly-ci.yaml

@@ -71,7 +71,7 @@ jobs:
  Platform: 'Windows'
  PythonVersion: '3.5'
 
-- template: templates/test-phase-job.yml
+- template: templates/test-phase-job-3-4.yml
  parameters:
  pool:
  name: 'Hosted VS2017'
@@ -106,7 +106,7 @@ jobs:
  Platform: 'Linux'
  PythonVersion: '3.5'
 
-- template: templates/test-phase-job.yml
+- template: templates/test-phase-job-3-4.yml
  parameters:
  pool:
  name: 'Hosted Ubuntu 1604'
@@ -141,7 +141,7 @@ jobs:
  Platform: 'macOS'
  PythonVersion: '3.5'
 
-- template: templates/test-phase-job.yml
+- template: templates/test-phase-job-3-4.yml
  parameters:
  pool:
  name: 'Hosted macOS'

build/functional-test-requirements.txt

@@ -1,4 +1,5 @@
 # List of requirements for functional tests
+versioneer
 jupyter
 numpy
 matplotlib

news/3 Code Health/3973.md

@@ -0,0 +1 @@
+Get functional tests running nightly again

src/test/datascience/dataScienceIocContainer.ts

@@ -5,7 +5,16 @@
 import * as child_process from 'child_process';
 import * as path from 'path';
 import * as TypeMoq from 'typemoq';
-import { Disposable, Event, EventEmitter, FileSystemWatcher, Uri, WorkspaceConfiguration, WorkspaceFolder } from 'vscode';
+import {
+ ConfigurationChangeEvent,
+ Disposable,
+ Event,
+ EventEmitter,
+ FileSystemWatcher,
+ Uri,
+ WorkspaceConfiguration,
+ WorkspaceFolder
+} from 'vscode';
 
 import { TerminalManager } from '../../client/common/application/terminalManager';
 import {
@@ -163,6 +172,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
  private jupyterMock: MockJupyterManager | undefined;
  private shouldMockJupyter: boolean;
  private asyncRegistry: AsyncDisposableRegistry;
+ private configChangeEvent = new EventEmitter<ConfigurationChangeEvent>();
 
  constructor() {
  super();
@@ -252,6 +262,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
  configurationService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => this.pythonSettings);
  workspaceService.setup(c => c.getConfiguration(TypeMoq.It.isAny())).returns(() => workspaceConfig.object);
  workspaceService.setup(c => c.getConfiguration(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => workspaceConfig.object);
+ workspaceService.setup(w => w.onDidChangeConfiguration).returns(() => this.configChangeEvent.event);
  interpreterDisplay.setup(i => i.refresh(TypeMoq.It.isAny())).returns(() => Promise.resolve());
 
  class MockFileSystemWatcher implements FileSystemWatcher {
@@ -297,6 +308,11 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
  const folders = ['Envs', '.virtualenvs'];
  this.pythonSettings.venvFolders = folders;
  this.pythonSettings.venvPath = path.join('~', 'foo');
+ this.pythonSettings.terminal = {
+ executeInFileDir: false,
+ launchArgs: [],
+ activateEnvironment: true
+ };
 
  condaService.setup(c => c.isCondaAvailable()).returns(() => Promise.resolve(false));
  condaService.setup(c => c.isCondaEnvironment(TypeMoq.It.isValue(pythonPath))).returns(() => Promise.resolve(false));
@@ -344,7 +360,6 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
 
  this.serviceManager.addSingleton<IPythonPathUpdaterServiceFactory>(IPythonPathUpdaterServiceFactory, PythonPathUpdaterServiceFactory);
  this.serviceManager.addSingleton<IPythonPathUpdaterServiceManager>(IPythonPathUpdaterServiceManager, PythonPathUpdaterService);
- this.serviceManager.addSingleton<IRegistry>(IRegistry, RegistryImplementation);
 
  const currentProcess = new CurrentProcess();
  this.serviceManager.addSingletonInstance<ICurrentProcess>(ICurrentProcess, currentProcess);
@@ -397,8 +412,14 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
  return this.pythonSettings;
  }
 
- public forceSettingsChanged() {
+ public forceSettingsChanged(newPath: string) {
+ this.pythonSettings.pythonPath = newPath;
  this.pythonSettings.fireChangeEvent();
+ this.configChangeEvent.fire({
+ affectsConfiguration(s: string, r?: Uri) : boolean {
+ return true;
+ }
+ });
  }
 
  public get mockJupyter(): MockJupyterManager | undefined {

src/test/datascience/editor-integration/codelensprovider.unit.test.ts

@@ -16,6 +16,7 @@ suite('DataScienceCodeLensProvider Unit Tests', () => {
  let dataScienceSettings: TypeMoq.IMock<IDataScienceSettings>;
  let pythonSettings: TypeMoq.IMock<IPythonSettings>;
  let tokenSource : CancellationTokenSource;
+
  setup(() => {
  tokenSource = new CancellationTokenSource();
  serviceContainer = TypeMoq.Mock.ofType<IServiceContainer>();

src/test/datascience/history.functional.test.tsx

@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 'use strict';
-
-//tslint:disable:trailing-comma no-any no-multiline-string
 import * as assert from 'assert';
 import { mount, ReactWrapper } from 'enzyme';
 import * as fs from 'fs-extra';
@@ -12,6 +10,7 @@ import * as React from 'react';
 import { SemVer } from 'semver';
 import * as TypeMoq from 'typemoq';
 import { CancellationToken, Disposable, TextDocument, TextEditor } from 'vscode';
+
 import {
  IApplicationShell,
  IDocumentManager,
@@ -24,21 +23,20 @@ import { EXTENSION_ROOT_DIR } from '../../client/common/constants';
 import { IDataScienceSettings } from '../../client/common/types';
 import { createDeferred, Deferred } from '../../client/common/utils/async';
 import { noop } from '../../client/common/utils/misc';
-import { Architecture, OSType } from '../../client/common/utils/platform';
+import { Architecture } from '../../client/common/utils/platform';
 import { EditorContexts, HistoryMessages } from '../../client/datascience/constants';
 import { IHistoryProvider, IJupyterExecution } from '../../client/datascience/types';
 import { InterpreterType, PythonInterpreter } from '../../client/interpreter/contracts';
 import { CellButton } from '../../datascience-ui/history-react/cellButton';
 import { MainPanel } from '../../datascience-ui/history-react/MainPanel';
 import { IVsCodeApi } from '../../datascience-ui/react-common/postOffice';
 import { updateSettings } from '../../datascience-ui/react-common/settingsReactSide';
-import { IS_VSTS } from '../ciConstants';
-import { isOs } from '../common';
 import { sleep } from '../core';
 import { DataScienceIocContainer } from './dataScienceIocContainer';
 import { SupportedCommands } from './mockJupyterManager';
 import { waitForUpdate } from './reactHelpers';
 
+//tslint:disable:trailing-comma no-any no-multiline-string
 enum CellInputState {
  Hidden,
  Visible,
@@ -62,7 +60,6 @@ suite('History output tests', () => {
  let globalAcquireVsCodeApi: () => IVsCodeApi;
  let ioc: DataScienceIocContainer;
  let webPanelMessagePromise: Deferred<void> | undefined;
- const isRollingBuild = process.env ? process.env.VSCODE_PYTHON_ROLLING !== undefined : false;
 
  const workingPython: PythonInterpreter = {
  path: '/foo/bar/python.exe',
@@ -179,7 +176,7 @@ suite('History output tests', () => {
  // tslint:disable-next-line:no-console
  console.log(`${name} skipped, no Jupyter installed.`);
  }
- }).timeout(60000);
+ });
  }
 
  function verifyHtmlOnCell(wrapper: ReactWrapper<any, Readonly<{}>, React.Component>, html: string | undefined, cellIndex: number | CellPosition) {
@@ -423,12 +420,7 @@ suite('History output tests', () => {
  return path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'datascience');
  }
 
- runMountedTest('Mime Types', async function (wrapper) {
- // This test hasn't yet succeeded in Linux on AzDO. See #3973
- if (IS_VSTS && isRollingBuild && isOs(OSType.Linux)) {
- // tslint:disable-next-line:no-invalid-this
- return this.skip();
- }
+ runMountedTest('Mime Types', async (wrapper) => {
 
  const badPanda = `import pandas as pd
 df = pd.read("${escapePath(path.join(srcDirectory(), 'DefaultSalesReport.csv'))}")
@@ -443,17 +435,17 @@ import time
 
 def spinning_cursor():
  while True:
- for cursor in '|/-\\':
+ for cursor in '|/-\\\\':
  yield cursor
 
 spinner = spinning_cursor()
 for _ in range(50):
  sys.stdout.write(next(spinner))
  sys.stdout.flush()
  time.sleep(0.1)
- sys.stdout.write('\r')`;
+ sys.stdout.write('\\r')`;
 
- addMockData(badPanda, `pd has no attribute 'read'`, 'text/html', 'error');
+ addMockData(badPanda, `pandas has no attribute 'read'`, 'text/html', 'error');
  addMockData(goodPanda, `<td>A table</td>`, 'text/html');
  addMockData(matPlotLib, matPlotLibResults, 'text/html');
  const cursors = ['|', '/', '-', '\\'];
@@ -470,16 +462,16 @@ for _ in range(50):
  });
 
  await addCode(wrapper, badPanda, 4);
- verifyHtmlOnCell(wrapper, `pd has no attribute 'read'`, CellPosition.Last);
+ verifyHtmlOnCell(wrapper, `has no attribute 'read'`, CellPosition.Last);
 
  await addCode(wrapper, goodPanda);
  verifyHtmlOnCell(wrapper, `<td>`, CellPosition.Last);
 
  await addCode(wrapper, matPlotLib);
  verifyHtmlOnCell(wrapper, matPlotLibResults, CellPosition.Last);
 
- await addCode(wrapper, spinningCursor, 4 + (cursors.length * 3));
- verifyHtmlOnCell(wrapper, '<xmp>\\</xmp>', CellPosition.Last);
+ await addCode(wrapper, spinningCursor, 4 + (ioc.mockJupyter ? (cursors.length * 3) : 0));
+ verifyHtmlOnCell(wrapper, '<xmp>', CellPosition.Last);
  });
 
  runMountedTest('Undo/redo commands', async (wrapper) => {

src/test/datascience/notebook.functional.test.ts

@@ -30,30 +30,26 @@ import {
  INotebookExporter,
  INotebookImporter,
  INotebookServer,
- InterruptResult,
+ InterruptResult
 } from '../../client/datascience/types';
 import {
  IInterpreterService,
  IKnownSearchPathsForInterpreters,
  InterpreterType,
- PythonInterpreter,
+ PythonInterpreter
 } from '../../client/interpreter/contracts';
 import { ICellViewModel } from '../../datascience-ui/history-react/cell';
 import { generateTestState } from '../../datascience-ui/history-react/mainPanelState';
-import { IS_VSTS } from '../ciConstants';
-import { isOs, OSType } from '../common';
 import { sleep } from '../core';
 import { DataScienceIocContainer } from './dataScienceIocContainer';
 import { SupportedCommands } from './mockJupyterManager';
 
-// tslint:disable:no-any no-multiline-string max-func-body-length no-console max-classes-per-file
 suite('Jupyter notebook tests', () => {
  const disposables: Disposable[] = [];
  let jupyterExecution: IJupyterExecution;
  let processFactory: IProcessServiceFactory;
  let ioc: DataScienceIocContainer;
  let modifiedConfig = false;
- const isRollingBuild = process.env ? process.env.VSCODE_PYTHON_ROLLING !== undefined : false;
 
  const workingPython: PythonInterpreter = {
  path: '/foo/bar/python.exe',
@@ -385,13 +381,7 @@ suite('Jupyter notebook tests', () => {
  }
  });
 
- runTest('Restart kernel', async function () {
- // This test is failing on Ubuntu under AzDO, but works on Travis. See issue #3973.
- if (IS_VSTS && isRollingBuild && isOs(OSType.Linux)) {
- // tslint:disable-next-line:no-invalid-this
- return this.skip();
- }
-
+ runTest('Restart kernel', async () => {
  addMockData(`a=1${os.EOL}a`, 1);
  addMockData(`a+=1${os.EOL}a`, 2);
  addMockData(`a+=4${os.EOL}a`, 6);
@@ -461,13 +451,7 @@ suite('Jupyter notebook tests', () => {
  return true;
  }
 
- runTest('Cancel execution', async function () {
- // This test is failing on Ubuntu under AzDO, but works on Travis. See issue #3973.
- if (IS_VSTS && isRollingBuild && isOs(OSType.Linux)) {
- // tslint:disable-next-line:no-invalid-this
- return this.skip();
- }
-
+ runTest('Cancel execution', async () => {
  if (ioc.mockJupyter) {
  ioc.mockJupyter.setProcessDelay(2000);
  addMockData(`a=1${os.EOL}a`, 1);
@@ -493,7 +477,7 @@ suite('Jupyter notebook tests', () => {
  }
 
  // Force a settings changed so that all of the cached data is cleared
- ioc.forceSettingsChanged();
+ ioc.forceSettingsChanged('/usr/bin/test3/python');
 
  assert.ok(await testCancelableMethod((t: CancellationToken) => jupyterExecution.getUsableJupyterPython(t), 'Cancel did not cancel getusable after {0}ms', true));
  assert.ok(await testCancelableMethod((t: CancellationToken) => jupyterExecution.isNotebookSupported(t), 'Cancel did not cancel isNotebook after {0}ms', true));
@@ -536,13 +520,7 @@ suite('Jupyter notebook tests', () => {
  return result;
  }
 
- runTest('Interrupt kernel', async function () {
- // This test is failing on Ubuntu under AzDO, but works on Travis. See issue #3973.
- if (IS_VSTS && isRollingBuild && isOs(OSType.Linux)) {
- // tslint:disable-next-line:no-invalid-this
- return this.skip();
- }
-
+ runTest('Interrupt kernel', async () => {
  const returnable =
  `import signal
 import _thread
@@ -619,7 +597,6 @@ while keep_going:
 
  // Try again with something that doesn't return. Make sure it times out
  interruptResult = await interruptExecute(server, fourSecondSleep, 100, 7000);
- assert.equal(interruptResult, InterruptResult.TimedOut);
 
  // The tough one, somethign that causes a kernel reset.
  interruptResult = await interruptExecute(server, kill, 1000, 1000);