Change the default cell marker (microsoft#7782)

* Change the default cell marker * Fix test failures Fix gather to use new comment Fix formatting on code comments from nls * Fix unit test too * Fix gather to replace #%%

Author: rchiodo

news/2 Fixes/7674.md

@@ -0,0 +1,2 @@
+Change the default cell marker to '# %%' instead of '#%%' to prevent linter errors in python files with markers.
+Also added a new setting to change this - 'python.dataScience.defaultCellMarker'.

package.json

@@ -1562,6 +1562,12 @@
  "description": "Regular expression used to identify code cells. All code until the next match is considered part of this cell. \nDefaults to '^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])' if left blank",
  "scope": "resource"
  },
+ "python.dataScience.defaultCellMarker": {
+ "type": "string",
+ "default": "# %%",
+ "description": "Cell marker used for delineating a cell in a python file.",
+ "scope": "resource"
+ },
  "python.dataScience.markdownRegularExpression": {
  "type": "string",
  "default": "^(#\\s*%%\\s*\\[markdown\\]|#\\s*\\<markdowncell\\>)",

package.nls.json

@@ -219,7 +219,7 @@
  "DataScience.exportingFormat": "Exporting {0}",
  "DataScience.exportCancel": "Cancel",
  "Common.canceled": "Canceled",
- "DataScience.importChangeDirectoryComment": "#%% Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting",
+ "DataScience.importChangeDirectoryComment": "{0} Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting",
  "DataScience.exportChangeDirectoryComment": "# Change directory to VSCode workspace root so that relative path loads work correctly. Turn this addition off with the DataScience.changeDirOnImportExport setting",
  "DataScience.interruptKernelStatus": "Interrupting IPython Kernel",
  "DataScience.restartKernelAfterInterruptMessage": "Interrupting the kernel timed out. Do you want to restart the kernel instead? All variables will be lost.",
@@ -341,7 +341,7 @@
  "DataScience.jupyterDebuggerInstallPtvsdYes": "Yes",
  "DataScience.jupyterDebuggerInstallPtvsdNo": "No",
  "DataScience.cellStopOnErrorFormatMessage": "{0} cells were canceled due to an error in the previous cell.",
- "DataScience.instructionComments": "# To add a new cell, type '#%%'\n# To add a new markdown cell, type '#%% [markdown]'\n",
+ "DataScience.instructionComments": "# To add a new cell, type '{0}'\n# To add a new markdown cell, type '{0} [markdown]'\n",
  "DataScience.scrollToCellTitleFormatMessage": "Go to [{0}]",
  "DataScience.remoteDebuggerNotSupported": "Debugging while attached to a remote server is not currently supported.",
  "DataScience.save": "Save notebook",

package.nls.nl.json

@@ -115,7 +115,7 @@
  "DataScience.exportingFormat": "Aan het exporteren {0}",
  "DataScience.exportCancel": "Annuleren",
  "Common.canceled": "Geannuleerd",
- "DataScience.importChangeDirectoryComment": "#%% De werkmap van de werkruimte root naar de ipynb-bestandslocatie veranderen. Schakel deze toevoeging uit met de instelling DataScience.changeDirOnImportExport",
+ "DataScience.importChangeDirectoryComment": "{0} De werkmap van de werkruimte root naar de ipynb-bestandslocatie veranderen. Schakel deze toevoeging uit met de instelling DataScience.changeDirOnImportExport",
  "DataScience.exportChangeDirectoryComment": "# De map wijzigen naar de VSCode-werktuimte root zodat de relatieve pad-ladingen correct werken. Schakel deze toevoeging uit met de instelling DataScience.changeDirOnImportExport",
  "DataScience.interruptKernelStatus": "IPython-kernel onderbreken",
  "DataScience.restartKernelAfterInterruptMessage": "Het onderbreken van de kernel duurde te lang. Wil je de kernel in plaats daarvan herstarten? Alle variabelen zullen verloren gaan.",

snippets/python.json

@@ -242,12 +242,12 @@
  },
  "add/new/cell": {
  "prefix": "add/new/cell",
- "body": "#%%",
+ "body": "# %%",
  "description": "Code snippet to add a new cell"
  },
  "mark/markdown": {
  "prefix": "mark/markdown",
- "body": "#%% [markdown]",
+ "body": "# %% [markdown]",
  "description": "Code snippet to add a new markdown cell"
  }
 }

src/client/common/types.ts

@@ -355,6 +355,7 @@ export interface IDataScienceSettings {
  runMagicCommands?: string;
  runStartupCommands: string;
  debugJustMyCode: boolean;
+ defaultCellMarker?: string;
 }
 
 export const IConfigurationService = Symbol('IConfigurationService');

src/client/common/utils/localize.ts

@@ -149,7 +149,7 @@ export namespace DataScience {
  export const runAllCellsLensCommandTitle = localize('python.command.python.datascience.runallcells.title', 'Run all cells');
  export const runAllCellsAboveLensCommandTitle = localize('python.command.python.datascience.runallcellsabove.title', 'Run above');
  export const runCellAndAllBelowLensCommandTitle = localize('python.command.python.datascience.runcellandallbelow.title', 'Run Below');
- export const importChangeDirectoryComment = localize('DataScience.importChangeDirectoryComment', '#%% Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting');
+ export const importChangeDirectoryComment = localize('DataScience.importChangeDirectoryComment', '{0} Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting');
  export const exportChangeDirectoryComment = localize('DataScience.exportChangeDirectoryComment', '# Change directory to VSCode workspace root so that relative path loads work correctly. Turn this addition off with the DataScience.changeDirOnImportExport setting');
 
  export const restartKernelMessage = localize('DataScience.restartKernelMessage', 'Do you want to restart the Jupter kernel? All variables will be lost.');
@@ -263,7 +263,7 @@ export namespace DataScience {
  export const jupyterDebuggerInstallPtvsdNo = localize('DataScience.jupyterDebuggerInstallPtvsdNo', 'No');
  export const cellStopOnErrorFormatMessage = localize('DataScience.cellStopOnErrorFormatMessage', '{0} cells were canceled due to an error in the previous cell.');
  export const scrollToCellTitleFormatMessage = localize('DataScience.scrollToCellTitleFormatMessage', 'Go to [{0}]');
- export const instructionComments = localize('DataScience.instructionComments', '# To add a new cell, type "#%%"\n# To add a new markdown cell, type "#%% [markdown]"\n');
+ export const instructionComments = localize('DataScience.instructionComments', '# To add a new cell, type "{0}"\n# To add a new markdown cell, type "{0} [markdown]"\n');
  export const invalidNotebookFileError = localize('DataScience.invalidNotebookFileError', 'Notebook is not in the correct format. Check the file for correct json.');
  export const invalidNotebookFileErrorFormat = localize('DataScience.invalidNotebookFileError', '{0} is not a valid notebook file. Check the file for correct json.');
  export const nativeEditorTitle = localize('DataScience.nativeEditorTitle', 'Notebook Editor');

src/client/datascience/constants.ts

@@ -277,12 +277,13 @@ export namespace Identifiers {
  export const SvgSizeTag = 'sizeTag={{0}, {1}}';
  export const InteractiveWindowIdentity = 'history://EC155B3B-DC18-49DC-9E99-9A948AA2F27B';
  export const InteractiveWindowIdentityScheme = 'history';
+ export const DefaultCodeCellMarker = '# %%';
 }
 
 export namespace CodeSnippits {
  export const ChangeDirectory = ['{0}', '{1}', 'import os', 'try:', '\tos.chdir(os.path.join(os.getcwd(), \'{2}\'))', '\tprint(os.getcwd())', 'except:', '\tpass', ''];
  export const ChangeDirectoryCommentIdentifier = '# ms-python.python added'; // Not translated so can compare.
- export const ImportIPython = '#%%\nfrom IPython import get_ipython\n\n';
+ export const ImportIPython = '{0}\nfrom IPython import get_ipython\n\n{1}';
  export const MatplotLibInitSvg = `import matplotlib\n%matplotlib inline\n${Identifiers.MatplotLibDefaultParams} = dict(matplotlib.rcParams)\n%config InlineBackend.figure_formats = 'svg', 'png'`;
  export const MatplotLibInitPng = `import matplotlib\n%matplotlib inline\n${Identifiers.MatplotLibDefaultParams} = dict(matplotlib.rcParams)\n%config InlineBackend.figure_formats = 'png'`;
 }

src/client/datascience/editor-integration/codewatcher.ts

@@ -22,7 +22,7 @@ import { StopWatch } from '../../common/utils/stopWatch';
 import { captureTelemetry, sendTelemetryEvent } from '../../telemetry';
 import { ICodeExecutionHelper } from '../../terminals/types';
 import { CellMatcher } from '../cellMatcher';
-import { Commands, Telemetry } from '../constants';
+import { Commands, Identifiers, Telemetry } from '../constants';
 import { ICodeLensFactory, ICodeWatcher, IDataScienceErrorHandler, IInteractiveWindowProvider } from '../types';
 
 @injectable()
@@ -268,9 +268,10 @@ export class CodeWatcher implements ICodeWatcher {
 
  public async addEmptyCellToBottom(): Promise<void> {
  const editor = this.documentManager.activeTextEditor;
+ const cellDelineator = this.defaultCellMarker;
  if (editor) {
  editor.edit((editBuilder) => {
- editBuilder.insert(new Position(editor.document.lineCount, 0), '\n\n#%%\n');
+ editBuilder.insert(new Position(editor.document.lineCount, 0), `\n\n${cellDelineator}\n`);
  });
 
  const newPosition = new Position(editor.document.lineCount + 3, 0); // +3 to account for the added spaces and to position after the new mark
@@ -286,6 +287,7 @@ export class CodeWatcher implements ICodeWatcher {
  const editor = this.documentManager.activeTextEditor;
  const cellMatcher = new CellMatcher();
  let index = 0;
+ const cellDelineator = this.defaultCellMarker;
 
  if (editor) {
  editor.edit((editBuilder) => {
@@ -295,14 +297,14 @@ export class CodeWatcher implements ICodeWatcher {
  if (cellMatcher.isCell(editor.document.lineAt(i).text)) {
  lastCell = false;
  index = i;
- editBuilder.insert(new Position(i, 0), '#%%\n\n');
+ editBuilder.insert(new Position(i, 0), `${cellDelineator}\n\n`);
  break;
  }
  }
 
  if (lastCell) {
  index = editor.document.lineCount;
- editBuilder.insert(new Position(editor.document.lineCount, 0), '\n#%%\n');
+ editBuilder.insert(new Position(editor.document.lineCount, 0), `\n${cellDelineator}\n`);
  }
  });
  }
@@ -313,6 +315,10 @@ export class CodeWatcher implements ICodeWatcher {
  .then(() => this.advanceToRange(new Range(newPosition, newPosition)));
  }
 
+ private get defaultCellMarker(): string {
+ return this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker;
+ }
+
  private onCodeLensFactoryUpdated(): void {
  // Update our code lenses.
  if (this.document) {
@@ -419,7 +425,7 @@ export class CodeWatcher implements ICodeWatcher {
 
  if (editor) {
  editor.edit((editBuilder) => {
- editBuilder.insert(new Position(currentRange.end.line + 1, 0), '\n\n#%%\n');
+ editBuilder.insert(new Position(currentRange.end.line + 1, 0), `\n\n${this.defaultCellMarker}\n`);
  });
  }
 

src/client/datascience/gather/gather.ts

@@ -15,6 +15,7 @@ import { Common } from '../../common/utils/localize';
 import { noop } from '../../common/utils/misc';
 import { CellMatcher } from '../cellMatcher';
 import { concatMultilineString } from '../common';
+import { Identifiers } from '../constants';
 import { CellState, ICell as IVscCell, IGatherExecution, INotebookExecutionLogger } from '../types';
 
 /**
@@ -79,9 +80,13 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg
  if (cell === undefined) {
  return '';
  }
+
+ // Get the default cell marker as we need to replace #%% with it.
+ const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker;
+
  // Call internal slice method
  const slices = this._executionSlicer.sliceAllExecutions(cell);
- const program = slices[0].cellSlices.reduce(concat, '');
+ const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : '';
 
  // Add a comment at the top of the file explaining what gather does
  const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n';