Tests for prompting to install missing ipykernel (#14266)

Author: DonJayamanne

.vscode/launch.json

@@ -177,6 +177,35 @@
  "preLaunchTask": "Compile",
  "skipFiles": ["<node_internals>/**"]
  },
+ {
+ "name": "DataScience tests in VS Code (*.vscode.ts)",
+ "type": "extensionHost",
+ "request": "launch",
+ "runtimeExecutable": "${execPath}",
+ "args": [
+ "${workspaceFolder}/src/test/datascience",
+ "--disable-extensions",
+ "--extensionDevelopmentPath=${workspaceFolder}",
+ "--extensionTestsPath=${workspaceFolder}/out/test"
+ ],
+ "env": {
+ "VSC_PYTHON_CI_TEST_GREP": "", // Modify this to run a subset of the single workspace tests
+ "VSC_PYTHON_CI_TEST_INVERT_GREP": "", // Initialize this to invert the grep (exclude tests with value defined in grep).
+ "CI_PYTHON_PATH": "<PythonPath>", // Initialize this to invert the grep (exclude tests with value defined in grep).
+ "VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE": "true",
+ "TEST_FILES_SUFFIX": "vscode.test"
+ },
+ "stopOnEntry": false,
+ "sourceMaps": true,
+ "outFiles": [
+ "${workspaceFolder}/out/**/*.js",
+ "!${workspaceFolder}/**/node_modules**/*"
+ ],
+ "preLaunchTask": "Compile",
+ "skipFiles": [
+ "<node_internals>/**"
+ ]
+ },
  {
  "name": "Tests (Multiroot, VS Code, *.test.ts)",
  "type": "extensionHost",

build/ci/templates/test_phases.yml

@@ -373,6 +373,23 @@ steps:
  buildPlatform: '$(Agent.Os)-Py$(pythonVersion)'
  buildConfiguration: 'SystemTests'
 
+
+ # Slow DataScience tests with VS Code
+ # Create a venv & register it as a kernel.
+ # These tests are slow hence will only run on linux.
+ # This env will be used to install ipykernel & test for prompts if ipykernel is missing & similar tests.
+ # Ensure this is registered as a kernel.
+ - bash: |
+ python -m venv .venvnokernel
+ source .venvnokernel/bin/activate
+
+ python -m pip install ipykernel
+ python -m ipykernel install --user --name .venvnokernel --display-name .venvnokernel
+ python -m pip uninstall ipykernel --yes
+ displayName: 'Prepare Virtual Env for Kernel Tests'
+ workingDirectory: $(Build.SourcesDirectory)/src/test/datascience
+ condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScienceInVSCode'))
+
  # Set the CI_PYTHON_PATH variable that forces VS Code system tests to use
  # the specified Python interpreter.
  #
@@ -438,18 +455,29 @@ steps:
  env:
  DISPLAY: :10
 
- # Run the single workspace tests in VS Code Insiders.
+ # Run the single workspace tests for DS in VS Code Insiders.
  - script: |
  npm run testDataScience
  continueOnError: true
  displayName: 'Run DataScience Tests in VSCode Insiders'
- condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScience'))
+ condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScience'), not(contains(variables['TestsToRun'], 'testDataScienceInVSCode')))
  env:
  DISPLAY: :10
  VSC_PYTHON_CI_TEST_VSC_CHANNEL: 'insiders'
  VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE: 'true'
  TEST_FILES_SUFFIX: 'ds.test'
 
+ # Run the single workspace tests for DS in VS Code Stable.
+ - script: |
+ npm run testDataScienceInVSCode
+ displayName: 'Run DataScience Tests in VSCode Stable'
+ condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScienceInVSCode'))
+ env:
+ DISPLAY: :10
+ VSC_PYTHON_CI_TEST_VSC_CHANNEL: 'stable'
+ VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE: 'true'
+ TEST_FILES_SUFFIX: 'vscode.test'
+
  # Upload the test results to Azure DevOps to facilitate test reporting in their UX.
  - task: PublishTestResults@2
  displayName: 'Publish single workspace tests results'

build/ci/vscode-python-ci.yaml

@@ -66,6 +66,10 @@ stages:
  'Debugger':
  TestsToRun: 'testDebugger'
  NeedsPythonTestReqs: true
+ 'DataScience in VSCode':
+ TestsToRun: 'testDataScienceInVSCode'
+ NeedsPythonTestReqs: true
+ NeedsPythonFunctionalReqs: true
  'Smoke':
  TestsToRun: 'testSmoke'
  NeedsPythonTestReqs: true

build/ci/vscode-python-pr-validation.yaml

@@ -58,6 +58,10 @@ stages:
  TestsToRun: 'testDataScience'
  NeedsPythonTestReqs: true
  NeedsPythonFunctionalReqs: true
+ 'DataScience in VSCode':
+ TestsToRun: 'testDataScienceInVSCode'
+ NeedsPythonTestReqs: true
+ NeedsPythonFunctionalReqs: true
  'Smoke':
  TestsToRun: 'testSmoke'
  NeedsPythonTestReqs: true

package.json

@@ -3472,6 +3472,8 @@
  "testJediLSP": "node ./out/test/languageServers/jedi/lspSetup.js && cross-env CODE_TESTS_WORKSPACE=src/test VSC_PYTHON_CI_TEST_GREP='Language Server:' node ./out/test/testBootstrap.js ./out/test/standardTest.js && node ./out/test/languageServers/jedi/lspTeardown.js",
  "pretestDataScience": "node ./out/test/datascience/dsTestSetup.js",
  "testDataScience": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_PYTHON_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=ds.test VSC_PYTHON_FORCE_LOGGING=1 VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.js ./out/test/standardTest.js",
+ "pretestDataScienceInVSCode": "node ./out/test/datascience/dsTestSetup.js",
+ "testDataScienceInVSCode": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_PYTHON_CI_TEST_VSC_CHANNEL=stable TEST_FILES_SUFFIX=vscode.test VSC_PYTHON_FORCE_LOGGING=1 VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.js ./out/test/standardTest.js",
  "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js",
  "testPerformance": "node ./out/test/testBootstrap.js ./out/test/performanceTest.js",
  "testSmoke": "node ./out/test/smokeTest.js",

src/client/datascience/interactive-ipynb/nativeEditorOldWebView.ts

@@ -51,7 +51,7 @@ import {
 import { NativeEditor } from './nativeEditor';
 import { NativeEditorSynchronizer } from './nativeEditorSynchronizer';
 
-enum AskForSaveResult {
+export enum AskForSaveResult {
  Yes,
  No,
  Cancel

src/test/datascience/.vscode/settings.json

@@ -13,7 +13,6 @@
  "python.formatting.provider": "yapf",
  "python.pythonPath": "python",
  "python.experiments.optInto": [
- "LocalZMQKernel - experiment",
  "NativeNotebook - experiment"
  ],
  // Do not set this to "Microsoft", else it will result in LS being downloaded on CI
@@ -22,8 +21,11 @@
  "python.languageServer": "Jedi",
  // Ensure auto save is off.
  "files.autoSave": "off",
+ //If enabled, ensure we save immediately.
+ "files.autoSaveDelay": 1,
  // We don't want jupyter to start when testing (DS functionality or anything else).
  "python.dataScience.disableJupyterAutoStart": true,
  "python.logging.level": "debug",
- "python.dataScience.alwaysTrustNotebooks": true // In UI tests we don't want prompts for `Do you want to trust thie nb...`
+ "python.dataScience.alwaysTrustNotebooks": true, // In UI tests we don't want prompts for `Do you want to trust thie nb...`
+ "python.dataScience.useNotebookEditor": true
 }

src/test/datascience/dsTestSetup.ts

@@ -2,64 +2,77 @@
 // Licensed under the MIT License.
 
 import * as fs from 'fs-extra';
+import { applyEdits, ModificationOptions, modify } from 'jsonc-parser';
 import * as path from 'path';
+import { IS_CI_SERVER } from '../ciConstants';
 import { EXTENSION_ROOT_DIR_FOR_TESTS } from '../constants';
-/**
- * Modify package.json to ensure VSC Notebooks have been setup so tests can run.
- * This is required because we modify package.json during runtime, hence we need to do the same thing for tests.
- */
 
-const packageJsonFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'package.json');
-const content = JSON.parse(fs.readFileSync(packageJsonFile).toString());
+const settingsFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src/test/datascience/.vscode/settings.json');
 
-// This code is temporary.
-if (
- !content.enableProposedApi ||
- !Array.isArray(content.contributes.notebookOutputRenderer) ||
- !Array.isArray(content.contributes.notebookProvider)
-) {
- content.enableProposedApi = true;
- content.contributes.notebookOutputRenderer = [
- {
- viewType: 'jupyter-notebook-renderer',
- displayName: 'Jupyter Notebook Renderer',
- mimeTypes: [
- 'application/geo+json',
- 'application/vdom.v1+json',
- 'application/vnd.dataresource+json',
- 'application/vnd.plotly.v1+json',
- 'application/vnd.vega.v2+json',
- 'application/vnd.vega.v3+json',
- 'application/vnd.vega.v4+json',
- 'application/vnd.vega.v5+json',
- 'application/vnd.vegalite.v1+json',
- 'application/vnd.vegalite.v2+json',
- 'application/vnd.vegalite.v3+json',
- 'application/vnd.vegalite.v4+json',
- 'application/x-nteract-model-debug+json',
- 'image/gif',
- 'image/png',
- 'image/jpeg',
- 'text/latex',
- 'text/vnd.plotly.v1+html'
- ]
- }
- ];
- content.contributes.notebookProvider = [
- {
- viewType: 'jupyter-notebook',
- displayName: 'Jupyter Notebook',
- selector: [
- {
- filenamePattern: '*.ipynb'
- }
- ]
- }
- ];
+function updateTestsForNativeNotebooks() {
+ /**
+ * Modify package.json to ensure VSC Notebooks have been setup so tests can run.
+ * This is required because we modify package.json during runtime, hence we need to do the same thing for tests.
+ */
+ const packageJsonFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'package.json');
+ const content = JSON.parse(fs.readFileSync(packageJsonFile).toString());
+
+ // This code is temporary.
+ if (
+ !content.enableProposedApi ||
+ !Array.isArray(content.contributes.notebookOutputRenderer) ||
+ !Array.isArray(content.contributes.notebookProvider)
+ ) {
+ content.enableProposedApi = true;
+ content.contributes.notebookProvider = [
+ {
+ viewType: 'jupyter-notebook',
+ displayName: 'Jupyter Notebook',
+ selector: [
+ {
+ filenamePattern: '*.ipynb'
+ }
+ ]
+ }
+ ];
+ }
+
+ // Update package.json to pick experiments from our custom settings.json file.
+ content.contributes.configuration.properties['python.experiments.optInto'].scope = 'resource';
+ content.contributes.configuration.properties['python.logging.level'].scope = 'resource';
+
+ fs.writeFileSync(packageJsonFile, JSON.stringify(content, undefined, 4));
+
+ updateSettings(true);
 }
 
-// Update package.json to pick experiments from our custom settings.json file.
-content.contributes.configuration.properties['python.experiments.optInto'].scope = 'resource';
-content.contributes.configuration.properties['python.logging.level'].scope = 'resource';
+function updateSettings(useNativeNotebooks: boolean) {
+ const modificationOptions: ModificationOptions = {
+ formattingOptions: {
+ tabSize: 4,
+ insertSpaces: true
+ }
+ };
+ let settingsJson = fs.readFileSync(settingsFile).toString();
+ const experiments = useNativeNotebooks ? ['NativeNotebook - experiment'] : [];
+ const autoSave = useNativeNotebooks ? 'off' : 'afterDelay';
+
+ settingsJson = applyEdits(
+ settingsJson,
+ modify(settingsJson, ['python.experiments.optInto'], experiments, modificationOptions)
+ );
+ settingsJson = applyEdits(settingsJson, modify(settingsJson, ['files.autoSave'], autoSave, modificationOptions));
+
+ fs.writeFileSync(settingsFile, settingsJson);
+}
+function updateTestsForOldNotebooks() {
+ updateSettings(false);
+}
 
-fs.writeFileSync(packageJsonFile, JSON.stringify(content, undefined, 4));
+if (!IS_CI_SERVER) {
+ // Noop.
+} else if (process.env.VSC_PYTHON_CI_TEST_VSC_CHANNEL === 'insiders') {
+ updateTestsForNativeNotebooks();
+} else {
+ updateTestsForOldNotebooks();
+}

src/test/datascience/helpers.ts

@@ -4,7 +4,19 @@
 'use strict';
 
 import { noop } from 'lodash';
+import * as path from 'path';
+import { Uri } from 'vscode';
+import { ICommandManager } from '../../client/common/application/types';
 import { IDataScienceSettings } from '../../client/common/types';
+import { Commands } from '../../client/datascience/constants';
+import {
+ AskForSaveResult,
+ NativeEditorOldWebView
+} from '../../client/datascience/interactive-ipynb/nativeEditorOldWebView';
+import { INotebookEditorProvider } from '../../client/datascience/types';
+import { IServiceContainer } from '../../client/ioc/types';
+import { CommandSource } from '../../client/testing/common/constants';
+import { waitForCondition } from '../common';
 
 // The default base set of data science settings to use
 export function defaultDataScienceSettings(): IDataScienceSettings {
@@ -61,3 +73,23 @@ export function writeDiffSnapshot(_snapshot: any, _prefix: string) {
  // snapshotCounter += 1;
  // fs.writeFile(file, JSON.stringify(diff), { encoding: 'utf-8' }).ignoreErrors();
 }
+
+export async function openNotebook(serviceContainer: IServiceContainer, ipynbFile: string, ignoreSaving = true) {
+ const cmd = serviceContainer.get<ICommandManager>(ICommandManager);
+ await cmd.executeCommand(Commands.OpenNotebook, Uri.file(ipynbFile), CommandSource.commandPalette);
+ const editorProvider = serviceContainer.get<INotebookEditorProvider>(INotebookEditorProvider);
+ await waitForCondition(
+ async () =>
+ editorProvider.editors.length > 0 &&
+ !!editorProvider.activeEditor &&
+ editorProvider.activeEditor.file.fsPath.endsWith(path.basename(ipynbFile)),
+ 30_000,
+ 'Notebook not opened'
+ );
+
+ if (ignoreSaving && editorProvider.activeEditor && editorProvider.activeEditor instanceof NativeEditorOldWebView) {
+ // We don't care about changes, no need to save them.
+ // tslint:disable-next-line: no-any
+ (editorProvider.activeEditor as any).askForSave = () => Promise.resolve(AskForSaveResult.No);
+ }
+}