feat(NODE-6773): add support for $lookup with automatic encryption (#4427)

Author: nbbeeken

.gitignore

@@ -99,6 +99,7 @@ test/lambda/env.json
 
 # files generated by tooling in drivers-evergreen-tools
 secrets-export.sh
+secrets-export.fish
 mo-expansion.sh
 mo-expansion.yml
 expansions.sh

etc/bash_to_fish.mjs

@@ -0,0 +1,39 @@
+import { createReadStream, promises as fs } from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline/promises';
+
+/**
+ * Takes an "exports" only bash script file
+ * and converts it to fish syntax.
+ * Will crash on any line that isn't:
+ * - a comment
+ * - an empty line
+ * - a bash 'set' call
+ * - export VAR=VAL
+ */
+
+const fileName = process.argv[2];
+const outFileName = path.basename(fileName, '.sh') + '.fish';
+const input = createReadStream(process.argv[2]);
+const lines = readline.createInterface({ input });
+const output = await fs.open(outFileName, 'w');
+
+for await (let line of lines) {
+ line = line.trim();
+
+ if (!line.startsWith('export ')) {
+ if (line.startsWith('#')) continue;
+ if (line === '') continue;
+ if (line.startsWith('set')) continue;
+ throw new Error('Cannot translate: ' + line);
+ }
+
+ const varVal = line.slice('export '.length);
+ const variable = varVal.slice(0, varVal.indexOf('='));
+ const value = varVal.slice(varVal.indexOf('=') + 1);
+ await output.appendFile(`set -x ${variable} ${value}\n`);
+}
+
+output.close();
+input.close();
+lines.close();

package-lock.json

@@ -96,7 +96,7 @@
  "js-yaml": "^4.1.0",
  "mocha": "^10.8.2",
  "mocha-sinon": "^2.1.2",
- "mongodb-client-encryption": "^6.2.0",
+ "mongodb-client-encryption": "^6.3.0",
  "mongodb-legacy": "^6.1.3",
  "nyc": "^15.1.0",
  "prettier": "^3.4.2",

package.json

@@ -239,6 +239,7 @@ export class AutoEncrypter {
  this._kmsProviders = options.kmsProviders || {};
 
  const mongoCryptOptions: MongoCryptOptions = {
+ enableMultipleCollinfo: true,
  cryptoCallbacks
  };
  if (options.schemaMap) {

src/client-side-encryption/auto_encrypter.ts

@@ -16,6 +16,7 @@ import { getSocks, type SocksLib } from '../deps';
 import { MongoOperationTimeoutError } from '../error';
 import { type MongoClient, type MongoClientOptions } from '../mongo_client';
 import { type Abortable } from '../mongo_types';
+import { type CollectionInfo } from '../operations/list_collections';
 import { Timeout, type TimeoutContext, TimeoutError } from '../timeout';
 import {
  addAbortListener,
@@ -205,11 +206,19 @@ export class StateMachine {
  const mongocryptdManager = executor._mongocryptdManager;
  let result: Uint8Array | null = null;
 
- while (context.state !== MONGOCRYPT_CTX_DONE && context.state !== MONGOCRYPT_CTX_ERROR) {
+ // Typescript treats getters just like properties: Once you've tested it for equality
+ // it cannot change. Which is exactly the opposite of what we use state and status for.
+ // Every call to at least `addMongoOperationResponse` and `finalize` can change the state.
+ // These wrappers let us write code more naturally and not add compiler exceptions
+ // to conditions checks inside the state machine.
+ const getStatus = () => context.status;
+ const getState = () => context.state;
+
+ while (getState() !== MONGOCRYPT_CTX_DONE && getState() !== MONGOCRYPT_CTX_ERROR) {
  options.signal?.throwIfAborted();
- debug(`[context#${context.id}] ${stateToString.get(context.state) || context.state}`);
+ debug(`[context#${context.id}] ${stateToString.get(getState()) || getState()}`);
 
- switch (context.state) {
+ switch (getState()) {
  case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: {
  const filter = deserialize(context.nextMongoOperation());
  if (!metaDataClient) {
@@ -218,22 +227,28 @@ export class StateMachine {
  );
  }
 
- const collInfo = await this.fetchCollectionInfo(
+ const collInfoCursor = this.fetchCollectionInfo(
  metaDataClient,
  context.ns,
  filter,
  options
  );
- if (collInfo) {
- context.addMongoOperationResponse(collInfo);
+
+ for await (const collInfo of collInfoCursor) {
+ context.addMongoOperationResponse(serialize(collInfo));
+ if (getState() === MONGOCRYPT_CTX_ERROR) break;
  }
 
+ if (getState() === MONGOCRYPT_CTX_ERROR) break;
+
  context.finishMongoOperation();
  break;
  }
 
  case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS: {
  const command = context.nextMongoOperation();
+ if (getState() === MONGOCRYPT_CTX_ERROR) break;
+
  if (!mongocryptdClient) {
  throw new MongoCryptError(
  'unreachable state machine state: entered MONGOCRYPT_CTX_NEED_MONGO_MARKINGS but mongocryptdClient is undefined'
@@ -283,22 +298,21 @@ export class StateMachine {
 
  case MONGOCRYPT_CTX_READY: {
  const finalizedContext = context.finalize();
- // @ts-expect-error finalize can change the state, check for error
- if (context.state === MONGOCRYPT_CTX_ERROR) {
- const message = context.status.message || 'Finalization error';
+ if (getState() === MONGOCRYPT_CTX_ERROR) {
+ const message = getStatus().message || 'Finalization error';
  throw new MongoCryptError(message);
  }
  result = finalizedContext;
  break;
  }
 
  default:
- throw new MongoCryptError(`Unknown state: ${context.state}`);
+ throw new MongoCryptError(`Unknown state: ${getState()}`);
  }
  }
 
- if (context.state === MONGOCRYPT_CTX_ERROR || result == null) {
- const message = context.status.message;
+ if (getState() === MONGOCRYPT_CTX_ERROR || result == null) {
+ const message = getStatus().message;
  if (!message) {
  debug(
  `unidentifiable error in MongoCrypt - received an error status from \`libmongocrypt\` but received no error message.`
@@ -527,29 +541,24 @@ export class StateMachine {
  * @param filter - A filter for the listCollections command
  * @param callback - Invoked with the info of the requested collection, or with an error
  */
- async fetchCollectionInfo(
+ fetchCollectionInfo(
  client: MongoClient,
  ns: string,
  filter: Document,
  options?: { timeoutContext?: TimeoutContext } & Abortable
- ): Promise<Uint8Array | null> {
+ ): AsyncIterable<CollectionInfo> {
  const { db } = MongoDBCollectionNamespace.fromString(ns);
 
  const cursor = client.db(db).listCollections(filter, {
  promoteLongs: false,
  promoteValues: false,
  timeoutContext:
  options?.timeoutContext && new CursorTimeoutContext(options?.timeoutContext, Symbol()),
- signal: options?.signal
+ signal: options?.signal,
+ nameOnly: false
  });
 
- // There is always exactly zero or one matching documents, so this should always exhaust the cursor
- // in a single batch. We call `toArray()` just to be safe and ensure that the cursor is always
- // exhausted and closed.
- const collections = await cursor.toArray();
-
- const info = collections.length > 0 ? serialize(collections[0]) : null;
- return info;
+ return cursor;
  }
 
  /**