Use a custom renderer to render png/jpeg (microsoft#12978)

For #12977

Author: DonJayamanne

news/3 Code Health/12977.md

@@ -0,0 +1 @@
+Custom renderers for `png/jpeg` images in `Notebooks`.

package.json

@@ -3153,6 +3153,8 @@
  "application/vnd.vegalite.v4+json",
  "application/x-nteract-model-debug+json",
  "image/gif",
+ "image/png",
+ "image/jpeg",
  "text/latex",
  "text/vnd.plotly.v1+html"
  ]

src/client/datascience/notebook/helpers/helpers.ts

@@ -107,6 +107,7 @@ export function createCellFromVSCNotebookCell(vscCell: NotebookCell, model: INot
 
  // Add the metadata back to the cell if we have any.
  // Refer to `addCellMetadata` to see how metadata is stored in VSC Cells.
+ // This metadata would exist if the user copied and pasted an existing cell.
  if (vscCell.metadata.custom?.vscodeMetadata) {
  cell.data = {
  ...cell.data,
@@ -120,17 +121,19 @@ export function createCellFromVSCNotebookCell(vscCell: NotebookCell, model: INot
 }
 
 /**
- * Updates the VSC Cell metadata with metadata from our cells.
- * If user exits without saving, then we have all metadata in VSC document.
- * This way when users copy a cell, we have everything in the old cell to create a duplicate of the Jupyter cell.
- * (Remember: VSC Cells less information compared to Jupyter cells).
+ * Stores the Jupyter Cell metadata into the VSCode Cells.
+ * This is used to facilitate:
+ * 1. When a user copies and pastes a cell, then the corresponding metadata is also copied across.
+ * 2. Diffing (VSC knows about metadata & stuff that contributes changes to a cell).
  */
 export function updateVSCNotebookCellMetadata(cellMetadata: NotebookCellMetadata, cell: ICell) {
  cellMetadata.custom = cellMetadata.custom ?? {};
  // tslint:disable-next-line: no-any
  const metadata: Record<string, any> = {};
  cellMetadata.custom.vscodeMetadata = metadata;
- const propertiesToClone = ['metadata', 'attachments', 'outputs'];
+ // We put this only for VSC to display in diff view.
+ // Else we don't use this.
+ const propertiesToClone = ['metadata', 'attachments'];
  propertiesToClone.forEach((propertyToClone) => {
  if (cell.data[propertyToClone]) {
  metadata[propertyToClone] = cloneDeep(cell.data[propertyToClone]);
@@ -146,6 +149,21 @@ export function getDefaultCodeLanguage(model: INotebookModel) {
 }
 
 export function createVSCNotebookCellDataFromCell(model: INotebookModel, cell: ICell): NotebookCellData | undefined {
+ if (cell.data.cell_type === 'raw') {
+ const rawCell = cell.data;
+ return {
+ cellKind: vscodeNotebookEnums.CellKind.Code,
+ language: 'raw',
+ metadata: {
+ custom: {
+ metadata: rawCell.metadata,
+ attachments: rawCell.attachments
+ }
+ },
+ outputs: [],
+ source: concatMultilineStringInput(cell.data.source)
+ };
+ }
  if (cell.data.cell_type !== 'code' && cell.data.cell_type !== 'markdown') {
  traceError(`Conversion of Cell into VS Code NotebookCell not supported ${cell.data.cell_type}`);
  return;
@@ -250,56 +268,23 @@ export function cellOutputToVSCCellOutput(output: nbformat.IOutput): CellOutput
  * @param {nbformat.IDisplayData} output
  * @returns {(CellDisplayOutput | undefined)}
  */
-function translateDisplayDataOutput(output: nbformat.IDisplayData): CellDisplayOutput | undefined {
- const mimeTypes = Object.keys(output.data || {});
+function translateDisplayDataOutput(
+ output: nbformat.IDisplayData | nbformat.IDisplayUpdate | nbformat.IExecuteResult
+): CellDisplayOutput | undefined {
  // If no mimeType data, then there's nothing to display.
- if (!mimeTypes.length) {
+ if (!Object.keys(output.data || {}).length) {
  return;
  }
- // If we have images, then process those images.
- // If we have PNG or JPEG images with a background, then add that background as HTML
  const data = { ...output.data };
- if (mimeTypes.some(isImagePngOrJpegMimeType) && shouldConvertImageToHtml(output) && !output.data['text/html']) {
- const mimeType = 'image/png' in data ? 'image/png' : 'image/jpeg';
- const metadata = output.metadata || {};
- const needsBackground = typeof metadata.needs_background === 'string';
- const backgroundColor = metadata.needs_background === 'light' ? 'white' : 'black';
- const divStyle = needsBackground ? `background-color:${backgroundColor};` : '';
- const imgSrc = `data:${mimeType};base64,${output.data[mimeType]}`;
-
- let height = '';
- let width = '';
- let imgStyle = '';
- if (metadata[mimeType] && typeof metadata[mimeType] === 'object') {
- // tslint:disable-next-line: no-any
- const imageMetadata = metadata[mimeType] as any;
- height = imageMetadata.height ? `height=${imageMetadata.height}` : '';
- width = imageMetadata.width ? `width=${imageMetadata.width}` : '';
- if (imageMetadata.unconfined === true) {
- imgStyle = `style="max-width:none"`;
- }
- }
-
- // Hack, use same classes as used in VSCode for images.
- // This is to maintain consistently in displaying images (if we hadn't used HTML).
- // See src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts
- data[
- 'text/html'
- ] = `<div class='display' style="overflow:scroll;${divStyle}"><img src="${imgSrc}" ${imgStyle} ${height} ${width}></div>`;
- }
+ // tslint:disable-next-line: no-any
+ const metadata = output.metadata ? ({ custom: output.metadata } as any) : undefined;
  return {
  outputKind: vscodeNotebookEnums.CellOutputKind.Rich,
- data
+ data,
+ metadata // Used be renderers & VS Code for diffing (it knows what has changed).
  };
 }
 
-function shouldConvertImageToHtml(output: nbformat.IDisplayData) {
- const metadata = output.metadata || {};
- return typeof metadata.needs_background === 'string' || metadata['image/png'] || metadata['image/jpeg'];
-}
-function isImagePngOrJpegMimeType(mimeType: string) {
- return mimeType === 'image/png' || mimeType === 'image/jpeg';
-}
 function translateStreamOutput(output: nbformat.IStream): CellStreamOutput | CellDisplayOutput {
  // Do not return as `CellOutputKind.Text`. VSC will not translate ascii output correctly.
  // Instead format the output as rich.

src/client/datascience/notebook/integration.ts

@@ -81,6 +81,8 @@ export class NotebookIntegration implements IExtensionSingleActivationService {
  'application/vnd.vegalite.v4+json',
  'application/x-nteract-model-debug+json',
  'image/gif',
+ 'image/png',
+ 'image/jpeg',
  'text/latex',
  'text/vnd.plotly.v1+html'
  ]

src/client/datascience/notebook/renderer.ts

@@ -20,12 +20,28 @@ export class NotebookOutputRenderer implements VSCNotebookOutputRenderer {
  public render(document: NotebookDocument, request: NotebookRenderRequest) {
  let outputToSend = request.output;
  if (request.output.outputKind === CellOutputKind.Rich && request.mimeType in request.output.data) {
+ // tslint:disable-next-line: no-any
+ const metadata: Record<string, any> = {};
+ // Send metadata only for the mimeType we are interested in.
+ const customMetadata = request.output.metadata?.custom;
+ if (customMetadata) {
+ if (customMetadata[request.mimeType]) {
+ metadata[request.mimeType] = customMetadata[request.mimeType];
+ }
+ if (customMetadata.needs_background) {
+ metadata.needs_background = customMetadata.needs_background;
+ }
+ if (customMetadata.unconfined) {
+ metadata.unconfined = customMetadata.unconfined;
+ }
+ }
  outputToSend = {
  ...request.output,
- // Send only what we need & ignore other mimetypes.
+ // Send only what we need & ignore other mimeTypes.
  data: {
  [request.mimeType]: request.output.data[request.mimeType]
- }
+ },
+ metadata
  };
  }
  const id = uuid();

src/datascience-ui/interactive-common/cellOutput.tsx

@@ -484,14 +484,14 @@ export class CellOutput extends React.Component<ICellOutputProps> {
  const transformedList = outputs.map(this.transformOutput.bind(this));
 
  transformedList.forEach((transformed, index) => {
- const mimetype = transformed.output.mimeType;
+ const mimeType = transformed.output.mimeType;
  if (isIPyWidgetOutput(transformed.output.mimeBundle)) {
  // Create a view for this output if not already there.
  this.renderWidget(transformed.output);
- } else if (mimetype && isMimeTypeSupported(mimetype)) {
+ } else if (mimeType && isMimeTypeSupported(mimeType)) {
  // If that worked, use the transform
  // Get the matching React.Component for that mimetype
- const Transform = getTransform(mimetype);
+ const Transform = getTransform(mimeType);
 
  let className = transformed.output.isText ? 'cell-output-text' : 'cell-output-html';
  className = transformed.output.isError ? `${className} cell-output-error` : className;
@@ -525,14 +525,14 @@ export class CellOutput extends React.Component<ICellOutputProps> {
  }
  }
  } else if (
- !mimetype ||
- mimetype.startsWith('application/scrapbook.scrap.') ||
- mimetype.startsWith('application/aml')
+ !mimeType ||
+ mimeType.startsWith('application/scrapbook.scrap.') ||
+ mimeType.startsWith('application/aml')
  ) {
  // Silently skip rendering of these mime types, render an empty div so the user sees the cell was executed.
  buffer.push(<div key={index}></div>);
  } else {
- const str: string = this.getUnknownMimeTypeFormatString().format(mimetype);
+ const str: string = this.getUnknownMimeTypeFormatString().format(mimeType);
  buffer.push(<div key={index}>{str}</div>);
  }
  });

src/datascience-ui/ipywidgets/README.md

@@ -1,4 +1,4 @@
-# Hosting ipywidgets in non-notebook contexxt
+# Hosting ipywidgets in non-notebook context
 
 - Much of the work is influenced by sample `web3` from https://github.com/jupyter-widgets/ipywidgets/blob/master/examples/web3
 - Displaying `ipywidgets` in non notebook context requires 3 things:
@@ -13,5 +13,5 @@
 - Thats what the code in this folder does (wraps the html manager + custom kernel connection)
 - Kernel messages from the extension are sent to this layer using the `postoffice`
 - Similarly messages from sent from html manager via the kernel are sent to the actual kernel via the postoffice.
-- However, the sequence and massaging of the kernel messages requires a lot of work. Baiscally majority of the message processing from `/node_modules/@jupyterlab/services/lib/kernel/*.js`
+- However, the sequence and massaging of the kernel messages requires a lot of work. Basically majority of the message processing from `/node_modules/@jupyterlab/services/lib/kernel/*.js`
  - Much of the message processing logic is borrowed from `comm.js`, `default.js`, `future.js`, `kernel.js` and `manager.js`.

src/datascience-ui/renderers/render.tsx

@@ -11,29 +11,79 @@ import { fixMarkdown } from '../interactive-common/markdownManipulation';
 import { getTransform } from '../interactive-common/transforms';
 
 export interface ICellOutputProps {
- output: nbformat.IOutput;
+ output: nbformat.IExecuteResult | nbformat.IDisplayData;
  mimeType: string;
 }
 
 export class CellOutput extends React.Component<ICellOutputProps> {
- // tslint:disable-next-line: no-any
  constructor(prop: ICellOutputProps) {
  super(prop);
  }
  public render() {
- const mimeBundle = this.props.output.data as nbformat.IMimeBundle; // NOSONAR
- let data: nbformat.MultilineString | JSONObject = mimeBundle[this.props.mimeType!];
+ const mimeBundle = this.props.output.data;
+ const data: nbformat.MultilineString | JSONObject = mimeBundle[this.props.mimeType!];
 
- // Fixup latex to make sure it has the requisite $$ around it
- if (this.props.mimeType! === 'text/latex') {
- data = fixMarkdown(concatMultilineStringOutput(data as nbformat.MultilineString), true);
+ switch (this.props.mimeType) {
+ case 'text/latex':
+ return this.renderLatex(data);
+ case 'image/png':
+ case 'image/jpeg':
+ return this.renderImage(mimeBundle, this.props.output.metadata);
+
+ default:
+ return this.renderLatex(data);
  }
+ }
+ /**
+ * Custom rendering of image/png and image/jpeg to handle custom Jupyter metadata.
+ * Behavior adopted from Jupyter lab.
+ */
+ // tslint:disable-next-line: no-any
+ private renderImage(mimeBundle: nbformat.IMimeBundle, metadata: Record<string, any> = {}) {
+ const mimeType = 'image/png' in mimeBundle ? 'image/png' : 'image/jpeg';
+
+ const imgStyle: Record<string, string | number> = {};
+ const divStyle: Record<string, string | number> = { overflow: 'scroll' }; // This is the default style used by Jupyter lab.
+ const imgSrc = `data:${mimeType};base64,${mimeBundle[mimeType]}`;
 
+ if (typeof metadata.needs_background === 'string') {
+ divStyle.backgroundColor = metadata.needs_background === 'light' ? 'white' : 'black';
+ }
+ // tslint:disable-next-line: no-any
+ const imageMetadata = metadata[mimeType] as Record<string, any> | undefined;
+ if (imageMetadata) {
+ if (imageMetadata.height) {
+ imgStyle.height = imageMetadata.height;
+ }
+ if (imageMetadata.width) {
+ imgStyle.width = imageMetadata.width;
+ }
+ if (imageMetadata.unconfined === true) {
+ imgStyle.maxWidth = 'none';
+ }
+ }
+
+ // Hack, use same classes as used in VSCode for images (keep things as similar as possible).
+ // This is to maintain consistently in displaying images (if we hadn't used HTML).
+ // See src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts
+ // tslint:disable: react-a11y-img-has-alt
+ return (
+ <div className={'display'} style={divStyle}>
+ <img src={imgSrc} style={imgStyle}></img>
+ </div>
+ );
+ }
+ private renderOutput(data: nbformat.MultilineString | JSONObject) {
  const Transform = getTransform(this.props.mimeType!);
  return (
  <div>
  <Transform data={data} />
  </div>
  );
  }
+ private renderLatex(data: nbformat.MultilineString | JSONObject) {
+ // Fixup latex to make sure it has the requisite $$ around it
+ data = fixMarkdown(concatMultilineStringOutput(data as nbformat.MultilineString), true);
+ return this.renderOutput(data);
+ }
 }

src/test/datascience/dsTestSetup.ts

@@ -38,6 +38,8 @@ if (
  'application/vnd.vegalite.v4+json',
  'application/x-nteract-model-debug+json',
  'image/gif',
+ 'image/png',
+ 'image/jpeg',
  'text/latex',
  'text/vnd.plotly.v1+html'
  ]