refactor: Refactor Upload Directory for Improved Reliability (#1867)

* build: Update engines from 8 to 10 Will update to 10 to 12 very soon, but this will prevent eslint issues today * refactor: Refactor Upload Directory for Improved Reliability * chore: Fix file naming * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: full Node 10 compatiblity * refactor: use defaults from sample comments * fix: Promise-related fixes * fix: add prefix items with dirname Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: d-goog

samples/package.json

@@ -4,7 +4,7 @@
  "license": "Apache-2.0",
  "author": "Google Inc.",
  "engines": {
- "node": ">=8"
+ "node": ">=10"
  },
  "repository": "googleapis/nodejs-storage",
  "private": true,

samples/uploadDirectory.js

@@ -19,7 +19,10 @@
 // description: Uploads full hierarchy of a local directory to a bucket.
 // usage: node files.js upload-directory <bucketName> <directoryPath>
 
-async function main(bucketName, directoryPath) {
+function main(
+ bucketName = 'your-unique-bucket-name',
+ directoryPath = './local/path/to/directory'
+) {
  // [START upload_directory]
  /**
  * TODO(developer): Uncomment the following lines before running the sample.
@@ -36,68 +39,53 @@ async function main(bucketName, directoryPath) {
  // Creates a client
  const storage = new Storage();
 
+ const {promisify} = require('util');
  const fs = require('fs');
  const path = require('path');
- const fileList = [];
 
- async function uploadDirectory() {
- // Get a list of files from the specified directory
- let dirCtr = 1;
- let itemCtr = 0;
- const pathDirName = path.dirname(directoryPath);
-
- getFiles(directoryPath);
-
- function getFiles(directory) {
- fs.readdir(directory, (err, items) => {
- dirCtr--;
- itemCtr += items.length;
- items.forEach(item => {
- const fullPath = path.join(directory, item);
- fs.stat(fullPath, (err, stat) => {
- itemCtr--;
- if (stat.isFile()) {
- fileList.push(fullPath);
- } else if (stat.isDirectory()) {
- dirCtr++;
- getFiles(fullPath);
- }
- if (dirCtr === 0 && itemCtr === 0) {
- onComplete();
- }
- });
- });
- });
+ const readdir = promisify(fs.readdir);
+ const stat = promisify(fs.stat);
+
+ async function* getFiles(directory = '.') {
+ for (const file of await readdir(directory)) {
+ const fullPath = path.join(directory, file);
+ const stats = await stat(fullPath);
+
+ if (stats.isDirectory()) {
+ yield* getFiles(fullPath);
+ }
+
+ if (stats.isFile()) {
+ yield fullPath;
+ }
  }
+ }
 
- async function onComplete() {
- const resp = await Promise.all(
- fileList.map(filePath => {
- let destination = path.relative(pathDirName, filePath);
- // If running on Windows
- if (process.platform === 'win32') {
- destination = destination.replace(/\\/g, '/');
- }
- return storage
- .bucket(bucketName)
- .upload(filePath, {destination})
- .then(
- uploadResp => ({fileName: destination, status: uploadResp[0]}),
- err => ({fileName: destination, response: err})
- );
- })
- );
-
- const successfulUploads =
- fileList.length - resp.filter(r => r.status instanceof Error).length;
- console.log(
- `${successfulUploads} files uploaded to ${bucketName} successfully.`
- );
+ async function uploadDirectory() {
+ const bucket = storage.bucket(bucketName);
+ let successfulUploads = 0;
+
+ for await (const filePath of getFiles(directoryPath)) {
+ try {
+ const dirname = path.dirname(directoryPath);
+ const destination = path.relative(dirname, filePath);
+
+ await bucket.upload(filePath, {destination});
+
+ console.log(`Successfully uploaded: ${filePath}`);
+ successfulUploads++;
+ } catch (e) {
+ console.error(`Error uploading ${filePath}:`, e);
+ }
  }
+
+ console.log(
+ `${successfulUploads} files uploaded to ${bucketName} successfully.`
+ );
  }
 
  uploadDirectory().catch(console.error);
  // [END upload_directory]
 }
 
-main(...process.argv.slice(2)).catch(console.error);
+main(...process.argv.slice(2));