sources/bigquery-new-row/bigquery-new-row.mjs
import crypto from "crypto"; import { isString } from "lodash-es"; import googleCloud from "../../google_cloud.app.mjs"; import common from "../common/bigquery.mjs"; export default { ...common, key: "google_cloud-bigquery-new-row", name: "BigQuery - New Row", description: "Emit new events when a new row is added to a table", version: "0.1.9", dedupe: "unique", type: "source", props: { ...common.props, tableId: { propDefinition: [ googleCloud, "tableId", ({ datasetId }) => ({ datasetId, }), ], }, uniqueKey: { type: "string", label: "Unique Key", description: "The name of a column in the table to use for deduplication. See [the docs](https://github.com/PipedreamHQ/pipedream/tree/master/components/google_cloud/sources/bigquery-new-row#technical-details) for more info.", async options(context) { const { page } = context; if (page !== 0) { return []; } const columnNames = await this._getColumnNames(); return columnNames.sort(); }, }, }, hooks: { ...common.hooks, async deploy() { await this._validateColumn(this.uniqueKey); const lastResultId = await this._getIdOfLastRow(this.getInitialEventCount()); this._setLastResultId(lastResultId); }, async activate() { if (this._getLastResultId()) { return; } await this._validateColumn(this.uniqueKey); const lastResultId = await this._getIdOfLastRow(); this._setLastResultId(lastResultId); }, }, methods: { ...common.methods, _getLastResultId() { return this.db.get("lastResultId"); }, _setLastResultId(lastResultId) { this.db.set("lastResultId", lastResultId); console.log(` Next scan of table '${this.tableId}' will start at ${this.uniqueKey}=${lastResultId} `); }, async _validateColumn(columnNameToValidate) { if (!isString(columnNameToValidate)) { throw new Error("columnNameToValidate must be a string"); } const columnNames = await this._getColumnNames(); if (!columnNames.includes(columnNameToValidate)) { throw new Error(`Nonexistent column: ${columnNameToValidate}`); } }, async _getColumnNames() { const table = this.googleCloud .getBigQueryClient() .dataset(this.datasetId) .table(this.tableId); const [ metadata, ] = await table.getMetadata(); const { fields } = metadata.schema; return fields.map(({ name }) => name); }, async _getIdOfLastRow(offset = 0) { const limit = offset + 1; const query = ` SELECT * FROM \`${this.tableId}\` ORDER BY \`${this.uniqueKey}\` DESC LIMIT @limit `; const queryOpts = { query, params: { limit, }, }; const client = this.googleCloud .getBigQueryClient() .dataset(this.datasetId); const [ job, ] = await client.createQueryJob(queryOpts); const [ rows, ] = await job.getQueryResults(); if (rows.length === 0) { console.log(` No records found in the target table, will start scanning from the beginning `); return; } const startingRow = rows.pop(); return startingRow[this.uniqueKey]; }, getQueryOpts() { const lastResultId = this._getLastResultId(); let query = `SELECT * FROM \`${this.tableId}\``; if (lastResultId) { query += ` WHERE \`${this.uniqueKey}\` >= @lastResultId`; } query += ` ORDER BY \`${this.uniqueKey}\` DESC`; const params = lastResultId ? { lastResultId, } : {}; return { query, params, }; }, generateMeta(row, ts) { const id = row[this.uniqueKey]; const summary = `New row: ${id}`; return { id, summary, ts, }; }, generateMetaForCollection(rows, ts) { const hash = crypto.createHash("sha1"); rows .map((i) => i[this.uniqueKey]) .map((i) => i.toString()) .forEach((i) => hash.update(i)); const id = hash.digest("base64"); const rowCount = rows.length; const entity = rowCount === 1 ? "row" : "rows"; const summary = `${rowCount} new ${entity}`; return { id, summary, ts, }; }, }, };
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.