feat(storage): add support for 'skipIfExists' option for downloadMany (#2526)

* feature added * fix * Added system test and sample test cases * copyright fix * build: fix path-to-regexp to older version due to node 14 requirement * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Remove unnecessary sample code * Manually modify README regards remove unnecessary sample code * added 'skipIfExists' option for downloadMany Node Transfer Manager: add support for 'skipIfExists' option for downloadMany --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: harsha-accenture <harshavardhan.s.r@accenture.com>

Author: 3 people

src/transfer-manager.ts

@@ -24,7 +24,7 @@ import {
 } from './file.js';
 import pLimit from 'p-limit';
 import * as path from 'path';
-import {createReadStream, promises as fsp} from 'fs';
+import {createReadStream, existsSync, promises as fsp} from 'fs';
 import {CRC32C} from './crc32c.js';
 import {GoogleAuth} from 'google-auth-library';
 import {XMLParser, XMLBuilder} from 'fast-xml-parser';
@@ -108,6 +108,7 @@ export interface DownloadManyFilesOptions {
  prefix?: string;
  stripPrefix?: string;
  passthroughOptions?: DownloadOptions;
+ skipIfExists?: boolean;
 }
 
 export interface DownloadFileInChunksOptions {
@@ -524,6 +525,8 @@ export class TransferManager {
  * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files.
  * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through
  * to each individual download operation.
+ * @property {boolean} [skipIfExists] Do not download the file if it already exists in
+ * the destination.
  *
  */
  /**
@@ -605,6 +608,12 @@ export class TransferManager {
  if (options.stripPrefix) {
  passThroughOptionsCopy.destination = file.name.replace(regex, '');
  }
+ if (
+ options.skipIfExists &&
+ existsSync(passThroughOptionsCopy.destination || '')
+ ) {
+ continue;
+ }
 
  promises.push(
  limit(async () => {

test/transfer-manager.ts

@@ -30,6 +30,7 @@ import {
  TransferManager,
  Storage,
  DownloadResponse,
+ DownloadManyFilesOptions,
 } from '../src/index.js';
 import assert from 'assert';
 import * as path from 'path';
@@ -195,6 +196,10 @@ describe('Transfer Manager', () => {
  });
 
  describe('downloadManyFiles', () => {
+ beforeEach(() => {
+ sandbox.stub(fs, 'existsSync').returns(true);
+ });
+
  it('calls download for each provided file', async () => {
  let count = 0;
  const firstFile = new File(bucket, 'first.txt');
@@ -276,6 +281,27 @@ describe('Transfer Manager', () => {
  await transferManager.downloadManyFiles([file], {passthroughOptions});
  });
 
+ it('does not download files that already exist locally when skipIfExists is true', async () => {
+ const firstFile = new File(bucket, 'first.txt');
+ sandbox.stub(firstFile, 'download').callsFake(options => {
+ assert.strictEqual(
+ (options as DownloadManyFilesOptions).skipIfExists,
+ 0
+ );
+ });
+ const secondFile = new File(bucket, 'second.txt');
+ sandbox.stub(secondFile, 'download').callsFake(options => {
+ assert.strictEqual(
+ (options as DownloadManyFilesOptions).skipIfExists,
+ 0
+ );
+ });
+
+ const files = [firstFile, secondFile];
+ const options = {skipIfExists: true};
+ await transferManager.downloadManyFiles(files, options);
+ });
+
  it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => {
  const options = {};
  const filename = 'first.txt';