fix(NODE-2995): Add shared metadata MongoClient (#2772)

Automatic client side encryption needs to perform metadata look ups
like listCollections. In situations where the connection pool size
is constrained or in full use it can be impossible for an operation
to proceed. Adding a separate client in these situations permits the
metadata look ups to proceed unblocking operations.

Author: nbbeeken

.evergreen/config.yml

@@ -470,6 +470,36 @@ functions:
             -p 8100 \
             -v \
             --fault revoked
+  run custom csfle tests:
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: src
+        script: |
+          ${PREPARE_SHELL}
+          cat <<EOT > prepare_client_encryption.sh
+          export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
+          export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
+          export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+          export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
+          export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
+          EOT
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        timeout_secs: 60
+        script: |
+          ${PREPARE_SHELL}
+
+          # Disable xtrace (just in case it was accidentally set).
+          set +x
+          . ./prepare_client_encryption.sh
+          rm -f ./prepare_client_encryption.sh
+
+          MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
   upload test results:
     - command: attach.xunit_results
       params:
@@ -1177,6 +1207,18 @@ tasks:
         vars:
           NODE_LTS_NAME: fermium
       - func: run mongosh integration tests
+  - name: run-custom-csfle-tests
+    tags:
+      - run-custom-csfle-tests
+    commands:
+      - func: install dependencies
+        vars:
+          NODE_LTS_NAME: erbium
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: '4.4'
+          TOPOLOGY: server
+      - func: run custom csfle tests
 buildvariants:
   - name: macos-1014-dubnium
     display_name: macOS 10.14 Node Dubnium
@@ -1504,6 +1546,11 @@ buildvariants:
     run_on: rhel70
     tasks:
       - run-checks
+  - name: ubuntu1804-custom-csfle-tests
+    display_name: Custom FLE Version Test
+    run_on: ubuntu1804-test
+    tasks:
+      - run-custom-csfle-tests
   - name: mongosh_integration_tests
     display_name: mongosh integration tests
     run_on: ubuntu1804-test

.evergreen/config.yml.in

@@ -513,6 +513,37 @@ functions:
             -v \
             --fault revoked
 
+  "run custom csfle tests":
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          cat <<EOT > prepare_client_encryption.sh
+          export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
+          export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
+          export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+          export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
+          export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
+          EOT
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        timeout_secs: 60
+        script: |
+          ${PREPARE_SHELL}
+
+          # Disable xtrace (just in case it was accidentally set).
+          set +x
+          . ./prepare_client_encryption.sh
+          rm -f ./prepare_client_encryption.sh
+
+          MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
+
   "upload test results":
     # Upload the xunit-format test results.
     - command: attach.xunit_results

.evergreen/generate_evergreen_tasks.js

@@ -521,6 +521,11 @@ BUILD_VARIANTS.push({
   display_name: 'lint',
   run_on: 'rhel70',
   tasks: ['run-checks']
+}, {
+  name: 'ubuntu1804-custom-csfle-tests',
+  display_name: 'Custom FLE Version Test',
+  run_on: 'ubuntu1804-test',
+  tasks: ['run-custom-csfle-tests']
 });
 
 // singleton build variant for mongosh integration tests
@@ -557,6 +562,28 @@ BUILD_VARIANTS.push({
   tasks: AWS_AUTH_TASKS
 });
 
+// special case for custom CSFLE test
+SINGLETON_TASKS.push({
+  name: 'run-custom-csfle-tests',
+  tags: ['run-custom-csfle-tests'],
+  commands: [
+    {
+      func: 'install dependencies',
+      vars: {
+        NODE_LTS_NAME: 'erbium',
+      },
+    },
+    {
+      func: 'bootstrap mongo-orchestration',
+      vars: {
+        VERSION: '4.4',
+        TOPOLOGY: 'server'
+      }
+    },
+    { func: 'run custom csfle tests' }
+  ]
+});
+
 const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
 fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
 fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);

.evergreen/run-custom-csfle-tests.sh

@@ -0,0 +1,51 @@
+#! /usr/bin/env bash
+
+# Initiail checks for running these tests
+if [ -z ${AWS_ACCESS_KEY_ID+omitted} ]; then echo "AWS_ACCESS_KEY_ID is unset" && exit 1; fi
+if [ -z ${AWS_SECRET_ACCESS_KEY+omitted} ]; then echo "AWS_SECRET_ACCESS_KEY is unset" && exit 1; fi
+if [ -z ${CSFLE_KMS_PROVIDERS+omitted} ]; then echo "CSFLE_KMS_PROVIDERS is unset" && exit 1; fi
+
+[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# Environment Variables:
+# CSFLE_GIT_REF - set the git reference to checkout for a custom CSFLE version
+# CDRIVER_GIT_REF - set the git reference to checkout for a custom CDRIVER version (this is for libbson)
+
+CSFLE_GIT_REF=${CSFLE_GIT_REF:-master}
+CDRIVER_GIT_REF=${CDRIVER_GIT_REF:-1.17.4}
+
+rm -rf csfle-deps-tmp
+mkdir -p csfle-deps-tmp
+pushd csfle-deps-tmp
+
+rm -rf libmongocrypt mongo-c-driver
+
+git clone https://github.com./mongodb/libmongocrypt.git
+pushd libmongocrypt
+git fetch --tags
+git checkout "$CSFLE_GIT_REF" -b csfle-custom
+popd # libmongocrypt
+
+git clone https://github.com./mongodb/mongo-c-driver.git
+pushd mongo-c-driver
+git fetch --tags
+git checkout "$CDRIVER_GIT_REF" -b cdriver-custom
+popd # mongo-c-driver
+
+pushd libmongocrypt/bindings/node
+
+source ./.evergreen/find_cmake.sh
+bash ./etc/build-static.sh
+
+popd # libmongocrypt/bindings/node
+popd # csfle-deps-tmp
+
+npm install
+
+cp -r csfle-deps-tmp/libmongocrypt/bindings/node node_modules/mongodb-client-encryption
+
+export MONGODB_URI=${MONGODB_URI}
+npx mocha test/functional/client_side_encryption

src/connection_string.ts

@@ -29,7 +29,7 @@ import { MongoCredentials } from './cmap/auth/mongo_credentials';
 import type { TagSet } from './sdam/server_description';
 import { Logger, LoggerLevelId } from './logger';
 import { PromiseProvider } from './promise_provider';
-import { createAutoEncrypter } from './operations/connect';
+import { Encrypter } from './encrypter';
 
 /**
  * Determines whether a provided address matches the provided parent domain in order
@@ -424,9 +424,7 @@ export function parseOptions(
   }
 
   checkTLSOptions(mongoOptions);
-  if (mongoClient && options.autoEncryption) {
-    mongoOptions.autoEncrypter = createAutoEncrypter(mongoClient, mongoOptions);
-  }
+
   if (options.promiseLibrary) PromiseProvider.set(options.promiseLibrary);
 
   if (mongoOptions.directConnection && typeof mongoOptions.srvHost === 'string') {
@@ -439,6 +437,12 @@ export function parseOptions(
   mongoOptions.userSpecifiedReplicaSet =
     objectOptions.has('replicaSet') || urlOptions.has('replicaSet');
 
+  if (mongoClient && mongoOptions.autoEncryption) {
+    Encrypter.checkForMongoCrypt();
+    mongoOptions.encrypter = new Encrypter(mongoClient, uri, options);
+    mongoOptions.autoEncrypter = mongoOptions.encrypter.autoEncrypter;
+  }
+
   return mongoOptions;
 }
 

src/deps.ts

@@ -1,6 +1,6 @@
 import { MongoError } from './error';
 import type { MongoClient } from './mongo_client';
-import type { Document } from './bson';
+import type { deserialize, Document, serialize } from './bson';
 import type { Callback } from './utils';
 
 function makeErrorModule(error: any) {
@@ -81,6 +81,10 @@ export type AutoEncryptionLoggerLevelId = typeof AutoEncryptionLoggerLevel[keyof
 
 /** @public */
 export interface AutoEncryptionOptions {
+  /** @internal */
+  bson?: { serialize: typeof serialize; deserialize: typeof deserialize };
+  /** @internal client for metadata lookups */
+  metadataClient?: MongoClient;
   /** A `MongoClient` used to fetch keys from a key vault */
   keyVaultClient?: MongoClient;
   /** The namespace where keys are stored in the key vault */
@@ -168,6 +172,8 @@ export interface AutoEncryptionOptions {
 
 /** @public */
 export interface AutoEncrypter {
+  // eslint-disable-next-line @typescript-eslint/no-misused-new
+  new (client: MongoClient, options: AutoEncryptionOptions): AutoEncrypter;
   init(cb: Callback): void;
   teardown(force: boolean, callback: Callback): void;
   encrypt(ns: string, cmd: Document, options: any, callback: Callback<Document>): void;