feat(chromium): large file uploads (#12860)

Author: yury-s

packages/playwright-core/src/client/connection.ts

@@ -32,6 +32,7 @@ import { Playwright } from './playwright';
 import { Electron, ElectronApplication } from './electron';
 import * as channels from '../protocol/channels';
 import { Stream } from './stream';
+import { WritableStream } from './writableStream';
 import { debugLogger } from '../utils/debugLogger';
 import { SelectorsOwner } from './selectors';
 import { Android, AndroidSocket, AndroidDevice } from './android';
@@ -269,6 +270,9 @@ export class Connection extends EventEmitter {
  case 'Worker':
  result = new Worker(parent, type, guid, initializer);
  break;
+ case 'WritableStream':
+ result = new WritableStream(parent, type, guid, initializer);
+ break;
  default:
  throw new Error('Missing type ' + type);
  }

packages/playwright-core/src/client/elementHandle.ts

@@ -26,6 +26,13 @@ import path from 'path';
 import { assert, isString, mkdirIfNeeded } from '../utils/utils';
 import * as api from '../../types/types';
 import * as structs from '../../types/structs';
+import { BrowserContext } from './browserContext';
+import { WritableStream } from './writableStream';
+import { pipeline } from 'stream';
+import { promisify } from 'util';
+import { debugLogger } from '../utils/debugLogger';
+
+const pipelineAsync = promisify(pipeline);
 
 export class ElementHandle<T extends Node = Node> extends JSHandle<T> implements api.ElementHandle {
  readonly _elementChannel: channels.ElementHandleChannel;
@@ -139,7 +146,16 @@ export class ElementHandle<T extends Node = Node> extends JSHandle<T> implements
  }
 
  async setInputFiles(files: string | FilePayload | string[] | FilePayload[], options: channels.ElementHandleSetInputFilesOptions = {}) {
- await this._elementChannel.setInputFiles({ files: await convertInputFiles(files), ...options });
+ const frame = await this.ownerFrame();
+ if (!frame)
+ throw new Error('Cannot set input files to detached element');
+ const converted = await convertInputFiles(files, frame.page().context());
+ if (converted.files) {
+ await this._elementChannel.setInputFiles({ files: converted.files, ...options });
+ } else {
+ debugLogger.log('api', 'switching to large files mode');
+ await this._elementChannel.setInputFilePaths({ ...converted, ...options });
+ }
  }
 
  async focus(): Promise<void> {
@@ -241,8 +257,35 @@ export function convertSelectOptionValues(values: string | api.ElementHandle | S
 }
 
 type SetInputFilesFiles = channels.ElementHandleSetInputFilesParams['files'];
-export async function convertInputFiles(files: string | FilePayload | string[] | FilePayload[]): Promise<SetInputFilesFiles> {
- const items: (string | FilePayload)[] = Array.isArray(files) ? files : [ files ];
+type InputFilesList = {
+ files?: SetInputFilesFiles;
+ localPaths?: string[];
+ streams?: channels.WritableStreamChannel[];
+};
+export async function convertInputFiles(files: string | FilePayload | string[] | FilePayload[], context: BrowserContext): Promise<InputFilesList> {
+ const items: (string | FilePayload)[] = Array.isArray(files) ? files.slice() : [ files ];
+
+ const sizeLimit = 50 * 1024 * 1024;
+ const hasLargeBuffer = items.find(item => typeof item === 'object' && item.buffer && item.buffer.byteLength > sizeLimit);
+ if (hasLargeBuffer)
+ throw new Error('Cannot set buffer larger than 50Mb, please write it to a file and pass its path instead.');
+
+ const stats = await Promise.all(items.filter(isString).map(item => fs.promises.stat(item as string)));
+ const hasLargeFile = !!stats.find(s => s.size > sizeLimit);
+ if (hasLargeFile) {
+ if (context._connection.isRemote()) {
+ const streams: channels.WritableStreamChannel[] = await Promise.all(items.map(async item => {
+ assert(isString(item));
+ const { writableStream: stream } = await context._channel.createTempFile({ name: path.basename(item) });
+ const writable = WritableStream.from(stream);
+ await pipelineAsync(fs.createReadStream(item), writable.stream());
+ return stream;
+ }));
+ return { streams };
+ }
+ return { localPaths: items as string[] };
+ }
+
  const filePayloads: SetInputFilesFiles = await Promise.all(items.map(async item => {
  if (typeof item === 'string') {
  return {
@@ -257,7 +300,7 @@ export async function convertInputFiles(files: string | FilePayload | string[] |
  };
  }
  }));
- return filePayloads;
+ return { files: filePayloads };
 }
 
 export function determineScreenshotType(options: { path?: string, type?: 'png' | 'jpeg' }): 'png' | 'jpeg' | undefined {

packages/playwright-core/src/client/frame.ts

@@ -31,6 +31,7 @@ import { LifecycleEvent, URLMatch, SelectOption, SelectOptionOptions, FilePayloa
 import { urlMatches } from './clientHelper';
 import * as api from '../../types/types';
 import * as structs from '../../types/structs';
+import { debugLogger } from '../utils/debugLogger';
 
 export type WaitForNavigationOptions = {
  timeout?: number,
@@ -355,7 +356,13 @@ export class Frame extends ChannelOwner<channels.FrameChannel> implements api.Fr
  }
 
  async setInputFiles(selector: string, files: string | FilePayload | string[] | FilePayload[], options: channels.FrameSetInputFilesOptions = {}): Promise<void> {
- await this._channel.setInputFiles({ selector, files: await convertInputFiles(files), ...options });
+ const converted = await convertInputFiles(files, this.page().context());
+ if (converted.files) {
+ await this._channel.setInputFiles({ selector, files: converted.files, ...options });
+ } else {
+ debugLogger.log('api', 'switching to large files mode');
+ await this._channel.setInputFilePaths({ selector, ...converted, ...options });
+ }
  }
 
  async type(selector: string, text: string, options: channels.FrameTypeOptions = {}) {

packages/playwright-core/src/client/writableStream.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Writable } from 'stream';
+import * as channels from '../protocol/channels';
+import { ChannelOwner } from './channelOwner';
+
+export class WritableStream extends ChannelOwner<channels.WritableStreamChannel> {
+ static from(Stream: channels.WritableStreamChannel): WritableStream {
+ return (Stream as any)._object;
+ }
+
+ constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.WritableStreamInitializer) {
+ super(parent, type, guid, initializer);
+ }
+
+ stream(): Writable {
+ return new WritableStreamImpl(this._channel);
+ }
+}
+
+class WritableStreamImpl extends Writable {
+ private _channel: channels.WritableStreamChannel;
+
+ constructor(channel: channels.WritableStreamChannel) {
+ super();
+ this._channel = channel;
+ }
+
+ override async _write(chunk: any, encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+ const error = await this._channel.write({ binary: chunk.toString('base64') }).catch(e => e);
+ callback(error || null);
+ }
+
+ override async _final(callback: (error?: Error | null) => void) {
+ // Stream might be destroyed after the connection was closed.
+ const error = await this._channel.close().catch(e => e);
+ callback(error || null);
+ }
+}

packages/playwright-core/src/dispatchers/browserContextDispatcher.ts

@@ -28,6 +28,10 @@ import { ArtifactDispatcher } from './artifactDispatcher';
 import { Artifact } from '../server/artifact';
 import { Request, Response } from '../server/network';
 import { TracingDispatcher } from './tracingDispatcher';
+import * as fs from 'fs';
+import * as path from 'path';
+import { createGuid } from '../utils/utils';
+import { WritableStreamDispatcher } from './writableStreamDispatcher';
 
 export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channels.BrowserContextChannel> implements channels.BrowserContextChannel {
  _type_EventTarget = true;
@@ -96,6 +100,15 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
  }));
  }
 
+ async createTempFile(params: channels.BrowserContextCreateTempFileParams, metadata?: channels.Metadata): Promise<channels.BrowserContextCreateTempFileResult> {
+ const dir = this._context._browser.options.artifactsDir;
+ const tmpDir = path.join(dir, 'upload-' + createGuid());
+ await fs.promises.mkdir(tmpDir);
+ this._context._tempDirs.push(tmpDir);
+ const file = fs.createWriteStream(path.join(tmpDir, params.name));
+ return { writableStream: new WritableStreamDispatcher(this._scope, file) };
+ }
+
  async setDefaultNavigationTimeoutNoReply(params: channels.BrowserContextSetDefaultNavigationTimeoutNoReplyParams) {
  this._context.setDefaultNavigationTimeout(params.timeout);
  }

packages/playwright-core/src/dispatchers/elementHandlerDispatcher.ts

@@ -22,6 +22,7 @@ import { DispatcherScope, existingDispatcher, lookupNullableDispatcher } from '.
 import { JSHandleDispatcher, serializeResult, parseArgument } from './jsHandleDispatcher';
 import { FrameDispatcher } from './frameDispatcher';
 import { CallMetadata } from '../server/instrumentation';
+import { WritableStreamDispatcher } from './writableStreamDispatcher';
 
 export class ElementHandleDispatcher extends JSHandleDispatcher implements channels.ElementHandleChannel {
  _type_ElementHandle = true;
@@ -143,7 +144,17 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements chann
  }
 
  async setInputFiles(params: channels.ElementHandleSetInputFilesParams, metadata: CallMetadata): Promise<void> {
- return await this._elementHandle.setInputFiles(metadata, params.files, params);
+ return await this._elementHandle.setInputFiles(metadata, { files: params.files }, params);
+ }
+
+ async setInputFilePaths(params: channels.ElementHandleSetInputFilePathsParams, metadata: CallMetadata): Promise<void> {
+ let { localPaths } = params;
+ if (!localPaths) {
+ if (!params.streams)
+ throw new Error('Neither localPaths nor streams is specified');
+ localPaths = params.streams.map(c => (c as WritableStreamDispatcher).path());
+ }
+ return await this._elementHandle.setInputFiles(metadata, { localPaths }, params);
  }
 
  async focus(params: channels.ElementHandleFocusParams, metadata: CallMetadata): Promise<void> {

packages/playwright-core/src/dispatchers/frameDispatcher.ts

@@ -21,6 +21,7 @@ import { ElementHandleDispatcher } from './elementHandlerDispatcher';
 import { parseArgument, serializeResult } from './jsHandleDispatcher';
 import { ResponseDispatcher, RequestDispatcher } from './networkDispatchers';
 import { CallMetadata } from '../server/instrumentation';
+import { WritableStreamDispatcher } from './writableStreamDispatcher';
 
 export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel> implements channels.FrameChannel {
  _type_Frame = true;
@@ -200,8 +201,18 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel> im
  return { values: await this._frame.selectOption(metadata, params.selector, elements, params.options || [], params) };
  }
 
- async setInputFiles(params: channels.FrameSetInputFilesParams, metadata: CallMetadata): Promise<void> {
- return await this._frame.setInputFiles(metadata, params.selector, params.files, params);
+ async setInputFiles(params: channels.FrameSetInputFilesParams, metadata: CallMetadata): Promise<channels.FrameSetInputFilesResult> {
+ return await this._frame.setInputFiles(metadata, params.selector, { files: params.files }, params);
+ }
+
+ async setInputFilePaths(params: channels.FrameSetInputFilePathsParams, metadata: CallMetadata): Promise<void> {
+ let { localPaths } = params;
+ if (!localPaths) {
+ if (!params.streams)
+ throw new Error('Neither localPaths nor streams is specified');
+ localPaths = params.streams.map(c => (c as WritableStreamDispatcher).path());
+ }
+ return await this._frame.setInputFiles(metadata, params.selector, { localPaths }, params);
  }
 
  async type(params: channels.FrameTypeParams, metadata: CallMetadata): Promise<void> {

packages/playwright-core/src/dispatchers/writableStreamDispatcher.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as channels from '../protocol/channels';
+import { Dispatcher, DispatcherScope } from './dispatcher';
+import * as fs from 'fs';
+import { createGuid } from '../utils/utils';
+
+export class WritableStreamDispatcher extends Dispatcher<{ guid: string, stream: fs.WriteStream }, channels.WritableStreamChannel> implements channels.WritableStreamChannel {
+ _type_WritableStream = true;
+ constructor(scope: DispatcherScope, stream: fs.WriteStream) {
+ super(scope, { guid: 'writableStream@' + createGuid(), stream }, 'WritableStream', {});
+ }
+
+ async write(params: channels.WritableStreamWriteParams): Promise<channels.WritableStreamWriteResult> {
+ const stream = this._object.stream;
+ await new Promise<void>((fulfill, reject) => {
+ stream.write(Buffer.from(params.binary, 'base64'), error => {
+ if (error)
+ reject(error);
+ else
+ fulfill();
+ });
+ });
+ }
+
+ async close() {
+ const stream = this._object.stream;
+ await new Promise<void>(fulfill => stream.end(fulfill));
+ }
+
+ path(): string {
+ return this._object.stream.path as string;
+ }
+}