sources/new-document/new-document.mjs
import common from "../common/base.mjs"; import { Timestamp } from "mongodb"; export default { ...common, key: "mongodb-new-document", name: "New Document", description: "Emit new an event when a new document is added to a collection", version: "0.0.12", type: "source", dedupe: "unique", props: { ...common.props, database: { propDefinition: [ common.props.mongodb, "database", ], }, collection: { propDefinition: [ common.props.mongodb, "collection", (c) => ({ database: c.database, }), ], }, timestampField: { type: "string", label: "Timestamp Field", description: "The key of a timestamp field, such as 'created_at' that is set to the current timestamp when a document is created.", }, timestampFieldType: { type: "string", label: "Timestamp Field Type", description: "The type of the timestamp field", default: "Timestamp", options: [ "Timestamp", "Integer", "ISO8601", ], }, }, hooks: { async deploy() { const client = await this.mongodb.getClient(); await this.processEvent(client, Date.now(), 25); await client.close(); }, }, methods: { ...common.methods, _getLastTs() { return this.db.get("lastTs"); }, _setLastTs(lastTs) { this.db.set("lastTs", lastTs); }, makeTs(timestamp) { if (this.timestampFieldType === "Integer") { return timestamp; } if (this.timestampFieldType === "Timestamp") { return this.convertToTimestamp(timestamp); } if (this.timestampFieldType === "ISO8601") { return new Date(timestamp); } return timestamp; }, getTs(timestamp) { if (this.timestampFieldType === "Integer") { return timestamp; } if (this.timestampFieldType === "Timestamp") { return timestamp.getHighBits() * 1_000; } if (this.timestampFieldType === "ISO8601") { return new Date(timestamp).getTime(); } return timestamp; }, convertToTimestamp(timestamp) { const bigIntValue = BigInt(timestamp.toString()); const seconds = Number(bigIntValue >> 32n); const increment = Number(bigIntValue & 0xFFFFFFFFn); return new Timestamp({ t: seconds, i: increment, }); }, async processEvent(client, eventTs, max) { const lastTs = this._getLastTs() || 0; let maxTs = lastTs; let count = 0; const collection = this.mongodb.getCollection(client, this.database, this.collection); const sort = { [this.timestampField]: -1, }; const query = { [this.timestampField]: { $gt: this.makeTs(lastTs), }, }; const documents = await collection .find(query) .sort(sort) .toArray(); const docs = []; for (const doc of documents) { const ts = this.getTs(doc[this.timestampField]); if (!(ts > lastTs) || (max && count >= max)) { break; } if (ts > maxTs) { maxTs = ts; } docs.push(doc); count++; } docs.reverse().forEach((doc) => this.emitEvent(doc, eventTs)); this._setLastTs(maxTs); }, generateMeta({ _id: id }, ts) { return { id, summary: `New Document ID ${JSON.stringify(id)}`, ts, }; }, }, };
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.