Google Cloud BigQuery bug fix

components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs

@@ -3,7 +3,7 @@ import utils from "../../common/utils.mjs";
 
 export default {
  name: "Bigquery Insert Rows",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/create-bucket/create-bucket.mjs

@@ -3,7 +3,7 @@ import storageClasses from "../../utils/storageClasses.mjs";
 
 export default {
  name: "Create Bucket",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs

@@ -12,7 +12,7 @@ export default {
  key: "google_cloud-create-scheduled-query",
  name: "Create Scheduled Query",
  description: "Creates a scheduled query in Google Cloud. [See the documentation](https://cloud.google.com/bigquery/docs/scheduling-queries)",
- version: "0.0.2",
+ version: "0.0.3",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/get-bucket/get-bucket.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "Get Bucket Metadata",
- version: "0.0.5",
+ version: "0.0.6",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/get-object/get-object.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "Get Object",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/list-buckets/list-buckets.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "List Buckets",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/logging-write-log/logging-write-log.mjs

@@ -3,7 +3,7 @@ import logSeverity from "../../utils/logSeverity.mjs";
 
 export default {
  name: "Logging - Write Log",
- version: "0.0.5",
+ version: "0.0.6",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/run-query/run-query.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "Run Query",
- version: "0.0.2",
+ version: "0.0.3",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/search-objects/search-objects.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "Search Objects",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs

@@ -1,8 +1,14 @@
 import googleCloud from "../../google_cloud.app.mjs";
+import {
+ ZonesClient,
+ ZoneOperationsClient,
+ InstancesClient,
+} from "@google-cloud/compute";
+import { ConfigurationError } from "@pipedream/platform";
 
 export default {
  name: "Switch Instance Boot Status",
- version: "0.0.3",
+ version: "0.0.4",
  annotations: {
  destructiveHint: true,
  openWorldHint: true,
@@ -14,19 +20,23 @@ export default {
  props: {
  googleCloud,
  zone: {
- propDefinition: [
- googleCloud,
- "zoneName",
- ],
+ label: "Zone",
+ description: "The unique zone name",
+ type: "string",
+ async options() {
+ const zones = await this.listZones();
+ return zones.map((item) => (item.name));
+ },
  },
  instanceName: {
- propDefinition: [
- googleCloud,
- "instanceName",
- ({ zone }) => ({
- zone,
- }),
- ],
+ label: "Instance Name",
+ description: "The unique instance name",
+ type: "string",
+ async options({ zone }) {
+ if (!zone) { return []; }
+ const instances = await this.listVmInstancesByZone(zone);
+ return instances.map((item) => (item.name));
+ },
  },
  newInstanceStatus: {
  label: "New Instance Status",
@@ -44,6 +54,70 @@ export default {
  default: false,
  },
  },
+ methods: {
+ zonesClient() {
+ return new ZonesClient(this.googleCloud.sdkParams());
+ },
+ zoneOperationsClient() {
+ return new ZoneOperationsClient(this.googleCloud.sdkParams());
+ },
+ instancesClient() {
+ return new InstancesClient(this.googleCloud.sdkParams());
+ },
+ async listZones() {
+ const zonesClient = this.zonesClient();
+ const sdkParams = this.googleCloud.sdkParams();
+ const [
+ zones,
+ ] = await zonesClient.list({
+ project: sdkParams.projectId,
+ });
+ return zones;
+ },
+ async waitOperation(operation) {
+ const operationsClient = this.zoneOperationsClient();
+ const sdkParams = this.googleCloud.sdkParams();
+ while (operation.status !== "DONE") {
+ [
+ operation,
+ ] = await operationsClient.wait({
+ operation: operation.name,
+ project: sdkParams.projectId,
+ zone: operation.zone.split("/").pop(),
+ });
+ }
+ return operation;
+ },
+ async listVmInstancesByZone(zone) {
+ const instancesClient = this.instancesClient();
+ const sdkParams = this.googleCloud.sdkParams();
+ const [
+ instances,
+ ] = await instancesClient.list({
+ project: sdkParams.projectId,
+ zone,
+ });
+ return instances;
+ },
+ async switchInstanceBootStatus(zone, instance, newStatus) {
+ if (![
+ "start",
+ "stop",
+ ].includes(newStatus)) {
+ throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'.");
+ }
+ const instancesClient = this.instancesClient();
+ const sdkParams = this.googleCloud.sdkParams();
+ const [
+ response,
+ ] = await instancesClient[newStatus]({
+ project: sdkParams.projectId,
+ zone,
+ instance,
+ });
+ return response.latestResponse;
+ },
+ },
  async run({ $ }) {
  const {
  zone,
@@ -52,14 +126,14 @@ export default {
  newInstanceStatus,
  } = this;
 
- let operation = await this.googleCloud.switchInstanceBootStatus(
+ let operation = await this.switchInstanceBootStatus(
  zone,
  instanceName,
  newInstanceStatus,
  );
 
  if (waitCompletion) {
- operation = await this.googleCloud.waitOperation(operation);
+ operation = await this.waitOperation(operation);
  }
 
  $.export("$summary", `Instance ${instanceName} boot status was set to ${newInstanceStatus}.`);

components/google_cloud/actions/upload-object/upload-object.mjs

@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
  name: "Upload An Object",
- version: "0.0.4",
+ version: "0.0.5",
  annotations: {
  destructiveHint: false,
  openWorldHint: true,

components/google_cloud/google_cloud.app.mjs

@@ -2,13 +2,7 @@
 import { Logging } from "@google-cloud/logging";
 import { Storage } from "@google-cloud/storage";
 import { BigQuery } from "@google-cloud/bigquery";
-import {
- ZonesClient,
- ZoneOperationsClient,
- InstancesClient,
-} from "@google-cloud/compute";
 import { v1 as bqdt } from "@google-cloud/bigquery-data-transfer";
-import { ConfigurationError } from "@pipedream/platform";
 
 export default {
  type: "app",
@@ -135,68 +129,6 @@ export default {
  storageClient() {
  return new Storage(this.sdkParams());
  },
- instancesClient() {
- return new InstancesClient(this.sdkParams());
- },
- zoneOperationsClient() {
- return new ZoneOperationsClient(this.sdkParams());
- },
- zonesClient() {
- return new ZonesClient(this.sdkParams());
- },
- async listVmInstancesByZone(zone) {
- const instancesClient = this.instancesClient();
- const sdkParams = this.sdkParams();
- const [
- instances,
- ] = await instancesClient.list({
- project: sdkParams.projectId,
- zone,
- });
- return instances;
- },
- async listZones() {
- const zonesClient = this.zonesClient();
- const sdkParams = this.sdkParams();
- const [
- zones,
- ] = await zonesClient.list({
- project: sdkParams.projectId,
- });
- return zones;
- },
- async switchInstanceBootStatus(zone, instance, newStatus) {
- if (![
- "start",
- "stop",
- ].includes(newStatus)) {
- throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'.");
- }
- const instancesClient = this.instancesClient();
- const sdkParams = this.sdkParams();
- const [
- response,
- ] = await instancesClient[newStatus]({
- project: sdkParams.projectId,
- zone,
- instance,
- });
- return response.latestResponse;
- },
- async waitOperation(operation) {
- const operationsClient = this.zoneOperationsClient();
- const sdkParams = this.sdkParams();
- while (operation.status !== "DONE") {
- [
- operation,
- ] = await operationsClient.wait({
- operation: operation.name,
- project: sdkParams.projectId,
- zone: operation.zone.split("/").pop(),
- });
- }
- return operation;
- },
  getBigQueryClient() {
  const credentials = this.authKeyJson();
  const { project_id: projectId } = credentials;

components/google_cloud/package.json

@@ -1,6 +1,6 @@
 {
  "name": "@pipedream/google_cloud",
- "version": "0.6.2",
+ "version": "0.6.3",
  "description": "Pipedream Google_cloud Components",
  "main": "google_cloud.app.mjs",
  "keywords": [

components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs

@@ -9,7 +9,7 @@ export default {
  // eslint-disable-next-line pipedream/source-name
  name: "BigQuery - New Row",
  description: "Emit new events when a new row is added to a table",
- version: "0.1.8",
+ version: "0.1.9",
  dedupe: "unique",
  type: "source",
  props: {
@@ -56,9 +56,6 @@ export default {
  const lastResultId = await this._getIdOfLastRow();
  this._setLastResultId(lastResultId);
  },
- deactivate() {
- this._setLastResultId(null);
- },
  },
  methods: {
  ...common.methods,
@@ -113,7 +110,18 @@ export default {
  limit,
  },
  };
- const rows = await this.getRowsForQuery(queryOpts, this.datasetId);
+ const client = this.googleCloud
+ .getBigQueryClient()
+ .dataset(this.datasetId);
+
+ const [
+ job,
+ ] = await client.createQueryJob(queryOpts);
+
+ const [
+ rows,
+ ] = await job.getQueryResults();
+
  if (rows.length === 0) {
  console.log(`
  No records found in the target table, will start scanning from the beginning
@@ -126,15 +134,16 @@ export default {
  },
  getQueryOpts() {
  const lastResultId = this._getLastResultId();
- const query = `
- SELECT *
- FROM \`${this.tableId}\`
- WHERE \`${this.uniqueKey}\` >= @lastResultId
- ORDER BY \`${this.uniqueKey}\` ASC
- `;
- const params = {
- lastResultId,
- };
+ let query = `SELECT * FROM \`${this.tableId}\``;
+ if (lastResultId) {
+ query += ` WHERE \`${this.uniqueKey}\` >= @lastResultId`;
+ }
+ query += ` ORDER BY \`${this.uniqueKey}\` DESC`;
+ const params = lastResultId
+ ? {
+ lastResultId,
+ }
+ : {};
  return {
  query,
  params,

components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs

@@ -8,7 +8,7 @@ export default {
  // eslint-disable-next-line pipedream/source-name
  name: "BigQuery - Query Results",
  description: "Emit new events with the results of an arbitrary query",
- version: "0.1.7",
+ version: "0.1.8",
  dedupe: "unique",
  type: "source",
  props: {