Add support for debugging on a remote jupyter server (microsoft#6549)

* First steps in remote support for debugging * Add comment to help mitigate mismatches * Change to local host for remote enable attach * Handle path mapping * Fix live share to attach * Fix guest debugging to not actually start the debugger * Add news entry * Fix localize test * Add remote test for debugging * Review feedback

Author: rchiodo

news/1 Enhancements/6379.md

@@ -0,0 +1 @@
+Add support for remote debugging of jupyter cells.

package.json

@@ -1400,6 +1400,12 @@
  "description": "When debugging a cell, stop on the first line if no other breakpoints in the current cell.",
  "scope": "resource"
  },
+ "python.dataScience.remoteDebuggerPort": {
+ "type": "number",
+ "default": -1,
+ "description": "When debugging a cell, open this port on the remote box. If -1 is specified, a random port between 8889 and 9000 will be attempted.",
+ "scope": "resource"
+ },
  "python.dataScience.textOutputLineLimit": {
  "type": "number",
  "default": 10000,

package.nls.json

@@ -302,5 +302,10 @@
  "DataScience.jupyterDataRateExceeded": "Cannot view variable because data rate exceeded. Please restart your server with a higher data rate limit. For example, --NotebookApp.iopub_data_rate_limit=10000000000.0",
  "DataScience.addCellBelowCommandTitle": "Add cell",
  "DataScience.debugCellCommandTitle": "Debug cell",
- "DataScience.variableExplorerDisabledDuringDebugging": "Variables are not available while debugging."
+ "DataScience.variableExplorerDisabledDuringDebugging": "Variables are not available while debugging.",
+ "DataScience.jupyterDebuggerNotInstalledError" : "Pip module ptvsd is required for debugging cells. You will need to install it to debug cells.",
+ "DataScience.jupyterDebuggerPortNotAvailableError" : "Port {0} cannot be opened for debugging. Please specify a different port in the remoteDebuggerPort setting.",
+ "DataScience.jupyterDebuggerPortBlockedError" : "Port {0} cannot be connected to for debugging. Please let port {0} through your firewall.",
+ "DataScience.jupyterDebuggerPortNotAvailableSearchError" : "Ports in the range {0}-{1} cannot be found for debugging. Please specify a port in the remoteDebuggerPort setting.",
+ "DataScience.jupyterDebuggerPortBlockedSearchError" : "A port cannot be connected to for debugging. Please let ports {0}-{1} through your firewall."
 }

src/client/common/types.ts

@@ -328,6 +328,7 @@ export interface IDataScienceSettings {
  stopOnFirstLineWhileDebugging?: boolean;
  textOutputLineLimit?: number;
  magicCommandsAsComments?: boolean;
+ remoteDebuggerPort?: number;
 }
 
 export const IConfigurationService = Symbol('IConfigurationService');

src/client/common/utils/localize.ts

@@ -229,6 +229,11 @@ export namespace DataScience {
  export const addCellBelowCommandTitle = localize('DataScience.addCellBelowCommandTitle', 'Add cell');
  export const debugCellCommandTitle = localize('DataScience.debugCellCommandTitle', 'Debug cell');
  export const variableExplorerDisabledDuringDebugging = localize('DataScience.variableExplorerDisabledDuringDebugging', 'Variables are not available while debugging.');
+ export const jupyterDebuggerNotInstalledError = localize('DataScience.jupyterDebuggerNotInstalledError', 'Pip module ptvsd is required for debugging cells. You will need to install it to debug cells.');
+ export const jupyterDebuggerPortNotAvailableError = localize('DataScience.jupyterDebuggerPortNotAvailableError', 'Port {0} cannot be opened for debugging. Please specify a different port in the remoteDebuggerPort setting.');
+ export const jupyterDebuggerPortBlockedError = localize('DataScience.jupyterDebuggerPortBlockedError', 'Port {0} cannot be connected to for debugging. Please let port {0} through your firewall.');
+ export const jupyterDebuggerPortNotAvailableSearchError = localize('DataScience.jupyterDebuggerPortNotAvailableSearchError', 'Ports in the range {0}-{1} cannot be found for debugging. Please specify a port in the remoteDebuggerPort setting.');
+ export const jupyterDebuggerPortBlockedSearchError = localize('DataScience.jupyterDebuggerPortBlockedSearchError', 'A port cannot be connected to for debugging. Please let ports {0}-{1} through your firewall.');
 }
 
 export namespace DebugConfigStrings {

src/client/datascience/constants.ts

@@ -54,7 +54,7 @@ export namespace RegExpValues {
  export const KernelSpecOutputRegEx = /^\s*(\S+)\s+(\S+)$/;
  // This next one has to be a string because uglifyJS isn't handling the groups. We use named-js-regexp to parse it
  // instead.
- export const UrlPatternRegEx = '(?<PREFIX>https?:\\/\\/)((\\(.+\\s+or\\s+(?<IP>.+)\\))|(?<LOCAL>[^\\s]+))(?<REST>:.+)' ;
+ export const UrlPatternRegEx = '(?<PREFIX>https?:\\/\\/)((\\(.+\\s+or\\s+(?<IP>.+)\\))|(?<LOCAL>[^\\s]+))(?<REST>:.+)';
  export interface IUrlPatternGroupType {
  LOCAL: string | undefined;
  PREFIX: string | undefined;
@@ -145,7 +145,7 @@ export enum Telemetry {
  WebviewMonacoStyleUpdate = 'DATASCIENCE.WEBVIEW_MONACO_STYLE_UPDATE',
  DataViewerFetchTime = 'DATASCIENCE.DATAVIEWER_FETCH_TIME',
  FindJupyterKernelSpec = 'DATASCIENCE.FIND_JUPYTER_KERNEL_SPEC'
- }
+}
 
 export namespace HelpLinks {
  export const PythonInteractiveHelpLink = 'https://aka.ms/pyaiinstall';
@@ -155,6 +155,8 @@ export namespace HelpLinks {
 export namespace Settings {
  export const JupyterServerLocalLaunch = 'local';
  export const IntellisenseTimeout = 300;
+ export const RemoteDebuggerPortBegin = 8889;
+ export const RemoteDebuggerPortEnd = 9000;
 }
 
 export namespace Identifiers {

src/client/datascience/interactive-window/interactiveWindow.ts

@@ -615,7 +615,7 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
  sendTelemetryEvent(Telemetry.RemoteAddCode);
 
  // Submit this item as new code.
- this.submitCode(args.code, args.file, args.line, args.id).ignoreErrors();
+ this.submitCode(args.code, args.file, args.line, args.id, undefined, undefined, args.debug).ignoreErrors();
  }
  }
 
@@ -642,7 +642,6 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
  if (this.jupyterServer) {
  await this.jupyterServer.restartKernel(this.generateDataScienceExtraSettings().jupyterInterruptTimeout);
  await this.addSysInfo(SysInfoReason.Restart);
- await this.jupyterDebugger.enableAttach(this.jupyterServer);
 
  // Compute if dark or not.
  const knownDark = await this.isDark();
@@ -702,7 +701,7 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
 
  // Activate the other side, and send as if came from a file
  this.interactiveWindowProvider.getOrCreateActive().then(_v => {
- this.shareMessage(InteractiveWindowMessages.RemoteAddCode, { code: info.code, file: Identifiers.EmptyFileName, line: 0, id: info.id, originator: this.id });
+ this.shareMessage(InteractiveWindowMessages.RemoteAddCode, { code: info.code, file: Identifiers.EmptyFileName, line: 0, id: info.id, originator: this.id, debug: false });
  }).ignoreErrors();
  }
  }
@@ -716,7 +715,7 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
  // Transmit this submission to all other listeners (in a live share session)
  if (!id) {
  id = uuid();
- this.shareMessage(InteractiveWindowMessages.RemoteAddCode, { code, file, line, id, originator: this.id });
+ this.shareMessage(InteractiveWindowMessages.RemoteAddCode, { code, file, line, id, originator: this.id, debug: debug !== undefined ? debug : false });
  }
 
  // Create a deferred object that will wait until the status is disposed
@@ -1054,11 +1053,6 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
  // Now try to create a notebook server
  this.jupyterServer = await this.jupyterExecution.connectToNotebookServer(options);
 
- // Enable debugging support if set
- if (this.jupyterServer) {
- await this.jupyterDebugger.enableAttach(this.jupyterServer);
- }
-
  // Before we run any cells, update the dark setting
  if (this.jupyterServer) {
  await this.jupyterServer.setMatplotLibStyle(knownDark);
@@ -1152,6 +1146,9 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
  // For a restart, tell our window to reset
  if (reason === SysInfoReason.Restart || reason === SysInfoReason.New) {
  this.postMessage(InteractiveWindowMessages.RestartKernel).ignoreErrors();
+ if (this.jupyterServer) {
+ this.jupyterDebugger.onRestart(this.jupyterServer);
+ }
  }
 
  this.logger.logInformation(`Sys info for ${this.id} ${reason} complete`);

src/client/datascience/interactive-window/interactiveWindowTypes.ts

@@ -62,7 +62,7 @@ export namespace InteractiveWindowMessages {
 
 // These are the messages that will mirror'd to guest/hosts in
 // a live share session
-export const InteractiveWindowRemoteMessages : string[] = [
+export const InteractiveWindowRemoteMessages: string[] = [
  InteractiveWindowMessages.AddedSysInfo,
  InteractiveWindowMessages.RemoteAddCode
 ];
@@ -94,6 +94,7 @@ export interface IExecuteInfo {
  id: string;
  file: string;
  line: number;
+ debug: boolean;
 }
 
 export interface IRemoteAddCode extends IExecuteInfo {
@@ -196,27 +197,27 @@ export class IInteractiveWindowMapping {
  public [InteractiveWindowMessages.Started]: never | undefined;
  public [InteractiveWindowMessages.AddedSysInfo]: IAddedSysInfo;
  public [InteractiveWindowMessages.RemoteAddCode]: IRemoteAddCode;
- public [InteractiveWindowMessages.Activate] : never | undefined;
+ public [InteractiveWindowMessages.Activate]: never | undefined;
  public [InteractiveWindowMessages.ShowDataViewer]: IShowDataViewer;
  public [InteractiveWindowMessages.GetVariablesRequest]: number;
  public [InteractiveWindowMessages.GetVariablesResponse]: IJupyterVariablesResponse;
  public [InteractiveWindowMessages.GetVariableValueRequest]: IJupyterVariable;
  public [InteractiveWindowMessages.GetVariableValueResponse]: IJupyterVariable;
  public [InteractiveWindowMessages.VariableExplorerToggle]: boolean;
- public [CssMessages.GetCssRequest] : IGetCssRequest;
- public [CssMessages.GetCssResponse] : IGetCssResponse;
- public [InteractiveWindowMessages.ProvideCompletionItemsRequest] : IProvideCompletionItemsRequest;
- public [InteractiveWindowMessages.CancelCompletionItemsRequest] : ICancelIntellisenseRequest;
- public [InteractiveWindowMessages.ProvideCompletionItemsResponse] : IProvideCompletionItemsResponse;
- public [InteractiveWindowMessages.ProvideHoverRequest] : IProvideHoverRequest;
- public [InteractiveWindowMessages.CancelHoverRequest] : ICancelIntellisenseRequest;
- public [InteractiveWindowMessages.ProvideHoverResponse] : IProvideHoverResponse;
- public [InteractiveWindowMessages.ProvideSignatureHelpRequest] : IProvideSignatureHelpRequest;
- public [InteractiveWindowMessages.CancelSignatureHelpRequest] : ICancelIntellisenseRequest;
- public [InteractiveWindowMessages.ProvideSignatureHelpResponse] : IProvideSignatureHelpResponse;
- public [InteractiveWindowMessages.AddCell] : IAddCell;
- public [InteractiveWindowMessages.EditCell] : IEditCell;
- public [InteractiveWindowMessages.RemoveCell] : IRemoveCell;
+ public [CssMessages.GetCssRequest]: IGetCssRequest;
+ public [CssMessages.GetCssResponse]: IGetCssResponse;
+ public [InteractiveWindowMessages.ProvideCompletionItemsRequest]: IProvideCompletionItemsRequest;
+ public [InteractiveWindowMessages.CancelCompletionItemsRequest]: ICancelIntellisenseRequest;
+ public [InteractiveWindowMessages.ProvideCompletionItemsResponse]: IProvideCompletionItemsResponse;
+ public [InteractiveWindowMessages.ProvideHoverRequest]: IProvideHoverRequest;
+ public [InteractiveWindowMessages.CancelHoverRequest]: ICancelIntellisenseRequest;
+ public [InteractiveWindowMessages.ProvideHoverResponse]: IProvideHoverResponse;
+ public [InteractiveWindowMessages.ProvideSignatureHelpRequest]: IProvideSignatureHelpRequest;
+ public [InteractiveWindowMessages.CancelSignatureHelpRequest]: ICancelIntellisenseRequest;
+ public [InteractiveWindowMessages.ProvideSignatureHelpResponse]: IProvideSignatureHelpResponse;
+ public [InteractiveWindowMessages.AddCell]: IAddCell;
+ public [InteractiveWindowMessages.EditCell]: IEditCell;
+ public [InteractiveWindowMessages.RemoveCell]: IRemoveCell;
  public [InteractiveWindowMessages.LoadOnigasmAssemblyRequest]: never | undefined;
  public [InteractiveWindowMessages.LoadOnigasmAssemblyResponse]: Buffer;
  public [InteractiveWindowMessages.LoadTmLanguageRequest]: never | undefined;

src/client/datascience/jupyter/jupyterConnection.ts

@@ -42,7 +42,7 @@ class JupyterConnectionWaiter {
  private fileSystem: IFileSystem;
  private notebook_dir: string;
  private getServerInfo: (cancelToken?: CancellationToken) => Promise<JupyterServerInfo[] | undefined>;
- private createConnection: (b: string, t: string, p: Disposable) => IConnection;
+ private createConnection: (b: string, t: string, h: string, p: Disposable) => IConnection;
  private launchResult: ObservableExecutionResult<string>;
  private cancelToken: CancellationToken | undefined;
  private stderr: string[] = [];
@@ -52,7 +52,7 @@ class JupyterConnectionWaiter {
  launchResult: ObservableExecutionResult<string>,
  notebookFile: string,
  getServerInfo: (cancelToken?: CancellationToken) => Promise<JupyterServerInfo[] | undefined>,
- createConnection: (b: string, t: string, p: Disposable) => IConnection,
+ createConnection: (b: string, t: string, h: string, p: Disposable) => IConnection,
  serviceContainer: IServiceContainer,
  cancelToken?: CancellationToken
  ) {
@@ -124,7 +124,8 @@ class JupyterConnectionWaiter {
  if (matchInfo) {
  const url = matchInfo.url;
  const token = matchInfo.token;
- this.resolveStartPromise(url, token);
+ const host = matchInfo.hostname;
+ this.resolveStartPromise(url, token, host);
  }
  }
 
@@ -158,7 +159,7 @@ class JupyterConnectionWaiter {
  }
 
  // Here we parsed the URL correctly
- this.resolveStartPromise(`${url.protocol}//${url.host}${url.pathname}`, `${url.searchParams.get('token')}`);
+ this.resolveStartPromise(`${url.protocol}//${url.host}${url.pathname}`, `${url.searchParams.get('token')}`, url.hostname);
  }
  }
 
@@ -187,11 +188,11 @@ class JupyterConnectionWaiter {
  }
  }
 
- private resolveStartPromise = (baseUrl: string, token: string) => {
+ private resolveStartPromise = (baseUrl: string, token: string, hostName: string) => {
  // tslint:disable-next-line: no-any
  clearTimeout(this.launchTimeout as any);
  if (!this.startPromise.rejected) {
- const connection = this.createConnection(baseUrl, token, this.launchResult);
+ const connection = this.createConnection(baseUrl, token, hostName, this.launchResult);
  const origDispose = connection.dispose.bind(connection);
  connection.dispose = () => {
  // Stop listening when we disconnect
@@ -216,12 +217,14 @@ class JupyterConnectionWaiter {
 export class JupyterConnection implements IConnection {
  public baseUrl: string;
  public token: string;
+ public hostName: string;
  public localLaunch: boolean;
  public localProcExitCode: number | undefined;
  private disposable: Disposable | undefined;
  private eventEmitter: EventEmitter<number> = new EventEmitter<number>();
- constructor(baseUrl: string, token: string, disposable: Disposable, childProc: ChildProcess | undefined) {
+ constructor(baseUrl: string, token: string, hostName: string, disposable: Disposable, childProc: ChildProcess | undefined) {
  this.baseUrl = baseUrl;
+ this.hostName = hostName;
  this.token = token;
  this.localLaunch = true;
  this.disposable = disposable;
@@ -251,7 +254,7 @@ export class JupyterConnection implements IConnection {
  notebookExecution,
  notebookFile,
  getServerInfo,
- (baseUrl: string, token: string, processDisposable: Disposable) => new JupyterConnection(baseUrl, token, processDisposable, notebookExecution.proc),
+ (baseUrl: string, token: string, hostName: string, processDisposable: Disposable) => new JupyterConnection(baseUrl, token, hostName, processDisposable, notebookExecution.proc),
  serviceContainer,
  cancelToken
  );