Skip to content

Commit 722aa0c

Browse files
nbbeekennbbeeken
committed
fix(NODE-2995): Add shared metadata MongoClient
Automatic client side encryption needs to perform metadata look ups like listCollections. In situations where the connection pool size is constrained or in full use it can be impossible for an operation to proceed. Adding a separate client in these situations permits the metadata look ups to proceed unblocking operations.
1 parent c529d43 commit 722aa0c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+969
-744
lines changed

‎.evergreen/config.yml

+47
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,36 @@ functions:
470470
-p 8100 \
471471
-v \
472472
--fault revoked
473+
run custom csfle tests:
474+
- command: shell.exec
475+
type: test
476+
params:
477+
silent: true
478+
working_dir: src
479+
script: |
480+
${PREPARE_SHELL}
481+
cat <<EOT > prepare_client_encryption.sh
482+
export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
483+
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
484+
export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
485+
export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
486+
export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
487+
export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
488+
EOT
489+
- command: shell.exec
490+
type: test
491+
params:
492+
working_dir: src
493+
timeout_secs: 60
494+
script: |
495+
${PREPARE_SHELL}
496+
497+
# Disable xtrace (just in case it was accidentally set).
498+
set +x
499+
. ./prepare_client_encryption.sh
500+
rm -f ./prepare_client_encryption.sh
501+
502+
MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
473503
upload test results:
474504
- command: attach.xunit_results
475505
params:
@@ -1177,6 +1207,18 @@ tasks:
11771207
vars:
11781208
NODE_LTS_NAME: fermium
11791209
- func: run mongosh integration tests
1210+
- name: run-custom-csfle-tests
1211+
tags:
1212+
- run-custom-csfle-tests
1213+
commands:
1214+
- func: install dependencies
1215+
vars:
1216+
NODE_LTS_NAME: erbium
1217+
- func: bootstrap mongo-orchestration
1218+
vars:
1219+
VERSION: '4.4'
1220+
TOPOLOGY: server
1221+
- func: run custom csfle tests
11801222
buildvariants:
11811223
- name: macos-1014-dubnium
11821224
display_name: macOS 10.14 Node Dubnium
@@ -1504,6 +1546,11 @@ buildvariants:
15041546
run_on: rhel70
15051547
tasks:
15061548
- run-checks
1549+
- name: ubuntu1804-custom-csfle-tests
1550+
display_name: Custom FLE Version Test
1551+
run_on: ubuntu1804-test
1552+
tasks:
1553+
- run-custom-csfle-tests
15071554
- name: mongosh_integration_tests
15081555
display_name: mongosh integration tests
15091556
run_on: ubuntu1804-test

‎.evergreen/config.yml.in

+31
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,37 @@ functions:
513513
-v \
514514
--fault revoked
515515

516+
"run custom csfle tests":
517+
- command: shell.exec
518+
type: test
519+
params:
520+
silent: true
521+
working_dir: "src"
522+
script: |
523+
${PREPARE_SHELL}
524+
cat <<EOT > prepare_client_encryption.sh
525+
export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
526+
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
527+
export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
528+
export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
529+
export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
530+
export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
531+
EOT
532+
- command: shell.exec
533+
type: test
534+
params:
535+
working_dir: "src"
536+
timeout_secs: 60
537+
script: |
538+
${PREPARE_SHELL}
539+
540+
# Disable xtrace (just in case it was accidentally set).
541+
set +x
542+
. ./prepare_client_encryption.sh
543+
rm -f ./prepare_client_encryption.sh
544+
545+
MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
546+
516547
"upload test results":
517548
# Upload the xunit-format test results.
518549
- command: attach.xunit_results

‎.evergreen/generate_evergreen_tasks.js

+27
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,11 @@ BUILD_VARIANTS.push({
521521
display_name: 'lint',
522522
run_on: 'rhel70',
523523
tasks: ['run-checks']
524+
}, {
525+
name: 'ubuntu1804-custom-csfle-tests',
526+
display_name: 'Custom FLE Version Test',
527+
run_on: 'ubuntu1804-test',
528+
tasks: ['run-custom-csfle-tests']
524529
});
525530

526531
// singleton build variant for mongosh integration tests
@@ -557,6 +562,28 @@ BUILD_VARIANTS.push({
557562
tasks: AWS_AUTH_TASKS
558563
});
559564

565+
// special case for custom CSFLE test
566+
SINGLETON_TASKS.push({
567+
name: 'run-custom-csfle-tests',
568+
tags: ['run-custom-csfle-tests'],
569+
commands: [
570+
{
571+
func: 'install dependencies',
572+
vars: {
573+
NODE_LTS_NAME: 'erbium',
574+
},
575+
},
576+
{
577+
func: 'bootstrap mongo-orchestration',
578+
vars: {
579+
VERSION: '4.4',
580+
TOPOLOGY: 'server'
581+
}
582+
},
583+
{ func: 'run custom csfle tests' }
584+
]
585+
});
586+
560587
const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
561588
fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
562589
fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);

‎.evergreen/run-custom-csfle-tests.sh

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#! /usr/bin/env bash
2+
3+
# Initiail checks for running these tests
4+
if [ -z ${AWS_ACCESS_KEY_ID+omitted} ]; then echo "AWS_ACCESS_KEY_ID is unset" && exit 1; fi
5+
if [ -z ${AWS_SECRET_ACCESS_KEY+omitted} ]; then echo "AWS_SECRET_ACCESS_KEY is unset" && exit 1; fi
6+
if [ -z ${CSFLE_KMS_PROVIDERS+omitted} ]; then echo "CSFLE_KMS_PROVIDERS is unset" && exit 1; fi
7+
8+
[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
9+
10+
set -o xtrace # Write all commands first to stderr
11+
set -o errexit # Exit the script with error if any of the commands fail
12+
13+
# Environment Variables:
14+
# CSFLE_GIT_REF - set the git reference to checkout for a custom CSFLE version
15+
# CDRIVER_GIT_REF - set the git reference to checkout for a custom CDRIVER version (this is for libbson)
16+
17+
CSFLE_GIT_REF=${CSFLE_GIT_REF:-master}
18+
CDRIVER_GIT_REF=${CDRIVER_GIT_REF:-1.17.4}
19+
20+
rm -rf csfle-deps-tmp
21+
mkdir -p csfle-deps-tmp
22+
pushd csfle-deps-tmp
23+
24+
rm -rf libmongocrypt mongo-c-driver
25+
26+
git clone https://github.com./mongodb/libmongocrypt.git
27+
pushd libmongocrypt
28+
git fetch --tags
29+
git checkout "$CSFLE_GIT_REF" -b csfle-custom
30+
popd # libmongocrypt
31+
32+
git clone https://github.com./mongodb/mongo-c-driver.git
33+
pushd mongo-c-driver
34+
git fetch --tags
35+
git checkout "$CDRIVER_GIT_REF" -b cdriver-custom
36+
popd # mongo-c-driver
37+
38+
pushd libmongocrypt/bindings/node
39+
40+
source ./.evergreen/find_cmake.sh
41+
bash ./etc/build-static.sh
42+
43+
popd # libmongocrypt/bindings/node
44+
popd # csfle-deps-tmp
45+
46+
npm install
47+
48+
cp -r csfle-deps-tmp/libmongocrypt/bindings/node node_modules/mongodb-client-encryption
49+
50+
export MONGODB_URI=${MONGODB_URI}
51+
npx mocha test/functional/client_side_encryption

‎src/connection_string.ts

+8-4
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import { MongoCredentials } from './cmap/auth/mongo_credentials';
2929
import type { TagSet } from './sdam/server_description';
3030
import { Logger, LoggerLevelId } from './logger';
3131
import { PromiseProvider } from './promise_provider';
32-
import { createAutoEncrypter } from './operations/connect';
32+
import { Encrypter } from './encrypter';
3333

3434
/**
3535
* Determines whether a provided address matches the provided parent domain in order
@@ -424,9 +424,7 @@ export function parseOptions(
424424
}
425425

426426
checkTLSOptions(mongoOptions);
427-
if (mongoClient && options.autoEncryption) {
428-
mongoOptions.autoEncrypter = createAutoEncrypter(mongoClient, mongoOptions);
429-
}
427+
430428
if (options.promiseLibrary) PromiseProvider.set(options.promiseLibrary);
431429

432430
if (mongoOptions.directConnection && typeof mongoOptions.srvHost === 'string') {
@@ -439,6 +437,12 @@ export function parseOptions(
439437
mongoOptions.userSpecifiedReplicaSet =
440438
objectOptions.has('replicaSet') || urlOptions.has('replicaSet');
441439

440+
if (mongoClient && mongoOptions.autoEncryption) {
441+
Encrypter.checkForMongoCrypt();
442+
mongoOptions.encrypter = new Encrypter(mongoClient, uri, options);
443+
mongoOptions.autoEncrypter = mongoOptions.encrypter.autoEncrypter;
444+
}
445+
442446
return mongoOptions;
443447
}
444448

‎src/deps.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { MongoError } from './error';
22
import type { MongoClient } from './mongo_client';
3-
import type { Document } from './bson';
3+
import type { deserialize, Document, serialize } from './bson';
44
import type { Callback } from './utils';
55

66
function makeErrorModule(error: any) {
@@ -81,6 +81,10 @@ export type AutoEncryptionLoggerLevelId = typeof AutoEncryptionLoggerLevel[keyof
8181

8282
/** @public */
8383
export interface AutoEncryptionOptions {
84+
/** @internal */
85+
bson?: { serialize: typeof serialize; deserialize: typeof deserialize };
86+
/** @internal client for metadata lookups */
87+
metadataClient?: MongoClient;
8488
/** A `MongoClient` used to fetch keys from a key vault */
8589
keyVaultClient?: MongoClient;
8690
/** The namespace where keys are stored in the key vault */
@@ -168,6 +172,8 @@ export interface AutoEncryptionOptions {
168172

169173
/** @public */
170174
export interface AutoEncrypter {
175+
// eslint-disable-next-line @typescript-eslint/no-misused-new
176+
new (client: MongoClient, options: AutoEncryptionOptions): AutoEncrypter;
171177
init(cb: Callback): void;
172178
teardown(force: boolean, callback: Callback): void;
173179
encrypt(ns: string, cmd: Document, options: any, callback: Callback<Document>): void;

0 commit comments

Comments
 (0)