diff --git a/src/client/csfle.rs b/src/client/csfle.rs index 1dcc863ab..d99273895 100644 --- a/src/client/csfle.rs +++ b/src/client/csfle.rs @@ -122,6 +122,16 @@ impl ClientState { if opts.bypass_query_analysis == Some(true) { builder = builder.bypass_query_analysis(); } + if let Some(key_cache_expiration) = opts.key_cache_expiration { + let expiration_ms: u64 = key_cache_expiration.as_millis().try_into().map_err(|_| { + Error::invalid_argument(format!( + "key_cache_expiration must not exceed {} milliseconds, got {:?}", + u64::MAX, + key_cache_expiration + )) + })?; + builder = builder.key_cache_expiration(expiration_ms)?; + } let crypt = builder.build()?; if opts.extra_option(&EO_CRYPT_SHARED_REQUIRED)? == Some(true) && crypt.shared_lib_version().is_none() diff --git a/src/client/csfle/client_builder.rs b/src/client/csfle/client_builder.rs index a9e99c3f7..489a16eaf 100644 --- a/src/client/csfle/client_builder.rs +++ b/src/client/csfle/client_builder.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use crate::{bson::Document, error::Result, options::ClientOptions, Client}; use super::options::AutoEncryptionOptions; @@ -101,6 +103,13 @@ impl EncryptedClientBuilder { self } + /// Set the duration of time after which the data encryption key cache should expire. Defaults + /// to 60 seconds if unset. + pub fn key_cache_expiration(mut self, expiration: impl Into>) -> Self { + self.enc_opts.key_cache_expiration = expiration.into(); + self + } + /// Constructs a new `Client` using automatic encryption. May perform DNS lookups and/or spawn /// mongocryptd as part of `Client` initialization. pub async fn build(self) -> Result { diff --git a/src/client/csfle/client_encryption.rs b/src/client/csfle/client_encryption.rs index 15832a6ff..05aadc751 100644 --- a/src/client/csfle/client_encryption.rs +++ b/src/client/csfle/client_encryption.rs @@ -3,6 +3,8 @@ mod create_data_key; mod encrypt; +use std::time::Duration; + use mongocrypt::{ctx::KmsProvider, Crypt}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; @@ -61,32 +63,44 @@ impl ClientEncryption { key_vault_namespace: Namespace, kms_providers: impl IntoIterator)>, ) -> Result { - let kms_providers = KmsProviders::new(kms_providers)?; - let crypt = Crypt::builder() - .kms_providers(&kms_providers.credentials_doc()?)? - .use_need_kms_credentials_state() - .retry_kms(true)? - .use_range_v2()? - .build()?; - let exec = CryptExecutor::new_explicit( - key_vault_client.weak(), - key_vault_namespace.clone(), - kms_providers, - )?; - let key_vault = key_vault_client - .database(&key_vault_namespace.db) - .collection_with_options( - &key_vault_namespace.coll, - CollectionOptions::builder() - .write_concern(WriteConcern::majority()) - .read_concern(ReadConcern::majority()) - .build(), - ); - Ok(ClientEncryption { - crypt, - exec, - key_vault, - }) + Self::builder(key_vault_client, key_vault_namespace, kms_providers).build() + } + + /// Initialize a builder to construct a [`ClientEncryption`]. Methods on + /// [`ClientEncryptionBuilder`] can be chained to set options. + /// + /// ```no_run + /// # use bson::doc; + /// # use mongocrypt::ctx::KmsProvider; + /// # use mongodb::client_encryption::ClientEncryption; + /// # use mongodb::error::Result; + /// # fn func() -> Result<()> { + /// # let kv_client = todo!(); + /// # let kv_namespace = todo!(); + /// # let local_key = doc! { }; + /// let enc = ClientEncryption::builder( + /// kv_client, + /// kv_namespace, + /// [ + /// (KmsProvider::Local, doc! { "key": local_key }, None), + /// (KmsProvider::Kmip, doc! { "endpoint": "localhost:5698" }, None), + /// ] + /// ) + /// .build()?; + /// # Ok(()) + /// # } + /// ``` + pub fn builder( + key_vault_client: Client, + key_vault_namespace: Namespace, + kms_providers: impl IntoIterator)>, + ) -> ClientEncryptionBuilder { + ClientEncryptionBuilder { + key_vault_client, + key_vault_namespace, + kms_providers: kms_providers.into_iter().collect(), + key_cache_expiration: None, + } } // pub async fn rewrap_many_data_key(&self, _filter: Document, _opts: impl @@ -189,6 +203,68 @@ impl ClientEncryption { } } +/// Builder for constructing a [`ClientEncryption`]. Construct by calling +/// [`ClientEncryption::builder`]. +pub struct ClientEncryptionBuilder { + key_vault_client: Client, + key_vault_namespace: Namespace, + kms_providers: Vec<(KmsProvider, bson::Document, Option)>, + key_cache_expiration: Option, +} + +impl ClientEncryptionBuilder { + /// Set the duration of time after which the data encryption key cache should expire. Defaults + /// to 60 seconds if unset. + pub fn key_cache_expiration(mut self, expiration: impl Into>) -> Self { + self.key_cache_expiration = expiration.into(); + self + } + + /// Build the [`ClientEncryption`]. + pub fn build(self) -> Result { + let kms_providers = KmsProviders::new(self.kms_providers)?; + + let mut crypt_builder = Crypt::builder() + .kms_providers(&kms_providers.credentials_doc()?)? + .use_need_kms_credentials_state() + .use_range_v2()? + .retry_kms(true)?; + if let Some(key_cache_expiration) = self.key_cache_expiration { + let expiration_ms: u64 = key_cache_expiration.as_millis().try_into().map_err(|_| { + Error::invalid_argument(format!( + "key_cache_expiration must not exceed {} milliseconds, got {:?}", + u64::MAX, + key_cache_expiration + )) + })?; + crypt_builder = crypt_builder.key_cache_expiration(expiration_ms)?; + } + let crypt = crypt_builder.build()?; + + let exec = CryptExecutor::new_explicit( + self.key_vault_client.weak(), + self.key_vault_namespace.clone(), + kms_providers, + )?; + let key_vault = self + .key_vault_client + .database(&self.key_vault_namespace.db) + .collection_with_options( + &self.key_vault_namespace.coll, + CollectionOptions::builder() + .write_concern(WriteConcern::majority()) + .read_concern(ReadConcern::majority()) + .build(), + ); + + Ok(ClientEncryption { + crypt, + exec, + key_vault, + }) + } +} + /// A KMS-specific key used to encrypt data keys. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] diff --git a/src/client/csfle/options.rs b/src/client/csfle/options.rs index f63993164..56247d945 100644 --- a/src/client/csfle/options.rs +++ b/src/client/csfle/options.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; use bson::Array; use mongocrypt::ctx::KmsProvider; @@ -8,6 +8,7 @@ use crate::{ bson::{Bson, Document}, client::options::TlsOptions, error::{Error, Result}, + serde_util, Namespace, }; @@ -59,6 +60,14 @@ pub(crate) struct AutoEncryptionOptions { #[cfg(test)] #[serde(skip)] pub(crate) disable_crypt_shared: Option, + /// The duration after which the data encryption key cache expires. Defaults to 60 seconds if + /// unset. + #[serde( + default, + rename = "keyExpirationMS", + deserialize_with = "serde_util::deserialize_duration_option_from_u64_millis" + )] + pub(crate) key_cache_expiration: Option, } fn default_key_vault_namespace() -> Namespace { @@ -81,6 +90,7 @@ impl AutoEncryptionOptions { bypass_query_analysis: None, #[cfg(test)] disable_crypt_shared: None, + key_cache_expiration: None, } } } diff --git a/src/cmap/conn.rs b/src/cmap/conn.rs index 681824471..85b9ade9d 100644 --- a/src/cmap/conn.rs +++ b/src/cmap/conn.rs @@ -327,8 +327,8 @@ impl PinnedConnectionHandle { } } - /// Retrieve the pinned connection. Will fail if the connection has been unpinned or is still in - /// use. + /// Retrieve the pinned connection. Will fail if the connection has been unpinned or is still + /// in use. pub(crate) async fn take_connection(&self) -> Result { use tokio::sync::mpsc::error::TryRecvError; let mut receiver = self.receiver.lock().await; diff --git a/src/test/spec/json/client-side-encryption/README.md b/src/test/spec/json/client-side-encryption/README.md index 1514c740d..304ff52ee 100644 --- a/src/test/spec/json/client-side-encryption/README.md +++ b/src/test/spec/json/client-side-encryption/README.md @@ -20,8 +20,8 @@ is supported when all of the following are true: - Server version is 4.2.0 or higher. Legacy spec test runners can rely on `runOn.minServerVersion` for this check. - Driver has libmongocrypt enabled -- At least one of [crypt_shared](../client-side-encryption.rst#crypt_shared) and/or - [mongocryptd](../client-side-encryption.rst#mongocryptd) is available. +- At least one of [crypt_shared](../client-side-encryption.md#crypt_shared) and/or + [mongocryptd](../client-side-encryption.md#mongocryptd) is available. ## Spec Test Format @@ -41,7 +41,7 @@ an array of BSON type strings is considered a match. For example, the following matches a command_started_event for an insert of a document where `random` must be of type `binData`: -``` +```text - command_started_event: command: insert: *collection_name @@ -54,7 +54,7 @@ For example, the following matches a command_started_event for an insert of a do The following matches a command_started_event for an insert of a document where `random` must be of type `binData` or `string`: -``` +```text - command_started_event: command: insert: *collection_name @@ -75,46 +75,46 @@ Each YAML file has the following keys: - `data` Unchanged from Transactions spec tests. - `json_schema` A JSON Schema that should be set on the collection (using `createCollection`) before each test run. - `encrypted_fields` An encryptedFields option that should be set on the collection (using `createCollection`) before - each test run. + each test run. - `key_vault_data` The data that should exist in the key vault collection under test before each test run. - `tests`: An array of tests that are to be run independently of each other. Each test will have some or all of the - following fields: - - `description`: Unchanged from Transactions spec tests. - - `skipReason`: Unchanged from Transactions spec tests. - - `useMultipleMongoses`: Unchanged from Transactions spec tests. - - `failPoint`: Unchanged from Transactions spec tests. - - `clientOptions`: Optional, parameters to pass to MongoClient(). - - `autoEncryptOpts`: Optional - - `kmsProviders` A dictionary of KMS providers to set on the key vault ("aws" or "local") - - `aws` The AWS KMS provider. An empty object. Drivers MUST fill in AWS credentials (`accessKeyId`, - `secretAccessKey`) from the environment. - - `azure` The Azure KMS provider credentials. An empty object. Drivers MUST fill in Azure credentials - (`tenantId`, `clientId`, and `clientSecret`) from the environment. - - `gcp` The GCP KMS provider credentials. An empty object. Drivers MUST fill in GCP credentials (`email`, - `privateKey`) from the environment. - - `local` or `local:name2` The local KMS provider. - - `key` A 96 byte local key. - - `kmip` The KMIP KMS provider credentials. An empty object. Drivers MUST fill in KMIP credentials (`endpoint`, - and TLS options). - - `schemaMap`: Optional, a map from namespaces to local JSON schemas. - - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". - - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. - Defaults to `false`. - - `encryptedFieldsMap` An optional document. The document maps collection namespace to `EncryptedFields` - documents. - - `operations`: Array of documents, each describing an operation to be executed. Each document has the following - fields: - - `name`: Unchanged from Transactions spec tests. - - `object`: Unchanged from Transactions spec tests.. Defaults to "collection" if omitted. - - `collectionOptions`: Unchanged from Transactions spec tests. - - `command_name`: Unchanged from Transactions spec tests. - - `arguments`: Unchanged from Transactions spec tests. - - `result`: Same as the Transactions spec test format with one addition: if the operation is expected to return an - error, the `result` document may contain an `isTimeoutError` boolean field. If `true`, the test runner MUST assert - that the error represents a timeout due to the use of the `timeoutMS` option. If `false`, the test runner MUST - assert that the error does not represent a timeout. - - `expectations`: Unchanged from Transactions spec tests. - - `outcome`: Unchanged from Transactions spec tests. + following fields: + - `description`: Unchanged from Transactions spec tests. + - `skipReason`: Unchanged from Transactions spec tests. + - `useMultipleMongoses`: Unchanged from Transactions spec tests. + - `failPoint`: Unchanged from Transactions spec tests. + - `clientOptions`: Optional, parameters to pass to MongoClient(). + - `autoEncryptOpts`: Optional + - `kmsProviders` A dictionary of KMS providers to set on the key vault ("aws" or "local") + - `aws` The AWS KMS provider. An empty object. Drivers MUST fill in AWS credentials (`accessKeyId`, + `secretAccessKey`) from the environment. + - `azure` The Azure KMS provider credentials. An empty object. Drivers MUST fill in Azure credentials + (`tenantId`, `clientId`, and `clientSecret`) from the environment. + - `gcp` The GCP KMS provider credentials. An empty object. Drivers MUST fill in GCP credentials (`email`, + `privateKey`) from the environment. + - `local` or `local:name2` The local KMS provider. + - `key` A 96 byte local key. + - `kmip` The KMIP KMS provider credentials. An empty object. Drivers MUST fill in KMIP credentials (`endpoint`, + and TLS options). + - `schemaMap`: Optional, a map from namespaces to local JSON schemas. + - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". + - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. + Defaults to `false`. + - `encryptedFieldsMap` An optional document. The document maps collection namespace to `EncryptedFields` + documents. + - `operations`: Array of documents, each describing an operation to be executed. Each document has the following + fields: + - `name`: Unchanged from Transactions spec tests. + - `object`: Unchanged from Transactions spec tests.. Defaults to "collection" if omitted. + - `collectionOptions`: Unchanged from Transactions spec tests. + - `command_name`: Unchanged from Transactions spec tests. + - `arguments`: Unchanged from Transactions spec tests. + - `result`: Same as the Transactions spec test format with one addition: if the operation is expected to return an + error, the `result` document may contain an `isTimeoutError` boolean field. If `true`, the test runner MUST + assert that the error represents a timeout due to the use of the `timeoutMS` option. If `false`, the test runner + MUST assert that the error does not represent a timeout. + - `expectations`: Unchanged from Transactions spec tests. + - `outcome`: Unchanged from Transactions spec tests. ## Credentials @@ -130,14 +130,15 @@ Test credentials to create environments are available in "drivers/gcpkms" and "d Do the following before running spec tests: -- If available for the platform under test, obtain a [crypt_shared](../client-side-encryption.rst#crypt_shared) binary - and place it in a location accessible to the tests. Refer to: [Using crypt_shared](#using-crypt_shared) +- If available for the platform under test, obtain a [crypt_shared](../client-side-encryption.md#crypt_shared) binary + and place it in a location accessible to the tests. Refer to: + [Using crypt_shared](../client-side-encryption.md#enabling-crypt_shared) - Start the mongocryptd process. - Start a mongod process with **server version 4.2.0 or later**. - Place credentials somewhere in the environment outside of tracked code. (If testing on evergreen, project variables - are a good place). + are a good place). - Start a KMIP test server on port 5698 by running - [drivers-evergreen-tools/.evergreen/csfle/kms_kmip_server.py](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_kmip_server.py). + [drivers-evergreen-tools/.evergreen/csfle/kms_kmip_server.py](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_kmip_server.py). Load each YAML (or JSON) file using a Canonical Extended JSON parser. @@ -150,187 +151,188 @@ Then for each element in `tests`: 2. If the `key_vault_data` field is present: - 1. Drop the `keyvault.datakeys` collection using writeConcern "majority". - 2. Insert the data specified into the `keyvault.datakeys` with write concern "majority". + 1. Drop the `keyvault.datakeys` collection using writeConcern "majority". + 2. Insert the data specified into the `keyvault.datakeys` with write concern "majority". 3. Create a MongoClient. 4. Create a collection object from the MongoClient, using the `database_name` and `collection_name` fields from the YAML - file. Drop the collection with writeConcern "majority". If a `json_schema` is defined in the test, use the - `createCollection` command to explicitly create the collection: + file. Drop the collection with writeConcern "majority". If a `json_schema` is defined in the test, use the + `createCollection` command to explicitly create the collection: - ```typescript - {"create": , "validator": {"$jsonSchema": }} - ``` + ```typescript + {"create": , "validator": {"$jsonSchema": }} + ``` - If `encrypted_fields` is defined in the test, the required collections and index described in - [Create and Drop Collection Helpers](../client-side-encryption.md#queryable-encryption-create-and-drop-collection-helpers) - must be created: + If `encrypted_fields` is defined in the test, the required collections and index described in + [Create and Drop Collection Helpers](../client-side-encryption.md#queryable-encryption-create-and-drop-collection-helpers) + must be created: - - Use the `dropCollection` helper with `encrypted_fields` as an option and writeConcern "majority". - - Use the `createCollection` helper with `encrypted_fields` as an option. + - Use the `dropCollection` helper with `encrypted_fields` as an option and writeConcern "majority". + - Use the `createCollection` helper with `encrypted_fields` as an option. 5. If the YAML file contains a `data` array, insert the documents in `data` into the test collection, using writeConcern - "majority". + "majority". 6. Create a **new** MongoClient using `clientOptions`. - 1. If `autoEncryptOpts` includes `aws`, `awsTemporary`, `awsTemporaryNoSessionToken`, `azure`, `gcp`, and/or `kmip` - as a KMS provider, pass in credentials from the environment. - - `awsTemporary`, and `awsTemporaryNoSessionToken` require temporary AWS credentials. These can be retrieved using - the csfle - [set-temp-creds.sh](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/csfle) - script. + 1. If `autoEncryptOpts` includes `aws`, `awsTemporary`, `awsTemporaryNoSessionToken`, `azure`, `gcp`, and/or `kmip` + as a KMS provider, pass in credentials from the environment. + - `awsTemporary`, and `awsTemporaryNoSessionToken` require temporary AWS credentials. These can be retrieved using + the csfle + [set-temp-creds.sh](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/csfle) + script. - - `aws`, `awsTemporary`, and `awsTemporaryNoSessionToken` are mutually exclusive. + - `aws`, `awsTemporary`, and `awsTemporaryNoSessionToken` are mutually exclusive. - `aws` should be substituted with: + `aws` should be substituted with: - ```javascript - "aws": { - "accessKeyId": , - "secretAccessKey": - } - ``` + ```javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + } + ``` - `awsTemporary` should be substituted with: + `awsTemporary` should be substituted with: - ```javascript - "aws": { - "accessKeyId": , - "secretAccessKey": - "sessionToken": - } - ``` + ```javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + "sessionToken": + } + ``` - `awsTemporaryNoSessionToken` should be substituted with: + `awsTemporaryNoSessionToken` should be substituted with: - ```javascript - "aws": { - "accessKeyId": , - "secretAccessKey": - } - ``` + ```javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + } + ``` - `gcp` should be substituted with: + `gcp` should be substituted with: - ```javascript - "gcp": { - "email": , - "privateKey": , - } - ``` + ```javascript + "gcp": { + "email": , + "privateKey": , + } + ``` - `azure` should be substituted with: + `azure` should be substituted with: - ```javascript - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - } - ``` + ```javascript + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + } + ``` - `local` should be substituted with: + `local` should be substituted with: - ```javascript - "local": { "key": } - ``` + ```javascript + "local": { "key": } + ``` - `kmip` should be substituted with: + `kmip` should be substituted with: - ```javascript - "kmip": { "endpoint": "localhost:5698" } - ``` + ```javascript + "kmip": { "endpoint": "localhost:5698" } + ``` - Configure KMIP TLS connections to use the following options: + Configure KMIP TLS connections to use the following options: - - `tlsCAFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). - This MAY be configured system-wide. - - `tlsCertificateKeyFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + - `tlsCAFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). + This MAY be configured system-wide. + - `tlsCertificateKeyFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + - The method of passing TLS options for KMIP TLS connections is driver dependent. - 2. If `autoEncryptOpts` does not include `keyVaultNamespace`, default it to `keyvault.datakeys`. + The method of passing TLS options for KMIP TLS connections is driver dependent. + 2. If `autoEncryptOpts` does not include `keyVaultNamespace`, default it to `keyvault.datakeys`. 7. For each element in `operations`: - - Enter a "try" block or your programming language's closest equivalent. + - Enter a "try" block or your programming language's closest equivalent. - - Create a Database object from the MongoClient, using the `database_name` field at the top level of the test file. + - Create a Database object from the MongoClient, using the `database_name` field at the top level of the test file. - - Create a Collection object from the Database, using the `collection_name` field at the top level of the test file. - If `collectionOptions` is present create the Collection object with the provided options. Otherwise create the - object with the default options. + - Create a Collection object from the Database, using the `collection_name` field at the top level of the test file. + If `collectionOptions` is present create the Collection object with the provided options. Otherwise create the + object with the default options. - - Execute the named method on the provided `object`, passing the arguments listed. + - Execute the named method on the provided `object`, passing the arguments listed. - - If the driver throws an exception / returns an error while executing this series of operations, store the error - message and server error code. + - If the driver throws an exception / returns an error while executing this series of operations, store the error + message and server error code. - - If the result document has an "errorContains" field, verify that the method threw an exception or returned an - error, and that the value of the "errorContains" field matches the error string. "errorContains" is a substring - (case-insensitive) of the actual error message. + - If the result document has an "errorContains" field, verify that the method threw an exception or returned an + error, and that the value of the "errorContains" field matches the error string. "errorContains" is a substring + (case-insensitive) of the actual error message. - If the result document has an "errorCodeName" field, verify that the method threw a command failed exception or - returned an error, and that the value of the "errorCodeName" field matches the "codeName" in the server error - response. + If the result document has an "errorCodeName" field, verify that the method threw a command failed exception or + returned an error, and that the value of the "errorCodeName" field matches the "codeName" in the server error + response. - If the result document has an "errorLabelsContain" field, verify that the method threw an exception or returned an - error. Verify that all of the error labels in "errorLabelsContain" are present in the error or exception using the - `hasErrorLabel` method. + If the result document has an "errorLabelsContain" field, verify that the method threw an exception or returned an + error. Verify that all of the error labels in "errorLabelsContain" are present in the error or exception using + the `hasErrorLabel` method. - If the result document has an "errorLabelsOmit" field, verify that the method threw an exception or returned an - error. Verify that none of the error labels in "errorLabelsOmit" are present in the error or exception using the - `hasErrorLabel` method. + If the result document has an "errorLabelsOmit" field, verify that the method threw an exception or returned an + error. Verify that none of the error labels in "errorLabelsOmit" are present in the error or exception using the + `hasErrorLabel` method. - - If the operation returns a raw command response, eg from `runCommand`, then compare only the fields present in the - expected result document. Otherwise, compare the method's return value to `result` using the same logic as the CRUD - Spec Tests runner. + - If the operation returns a raw command response, eg from `runCommand`, then compare only the fields present in the + expected result document. Otherwise, compare the method's return value to `result` using the same logic as the + CRUD Spec Tests runner. 8. If the test includes a list of command-started events in `expectations`, compare them to the actual command-started - events using the same logic as the - [Command Monitoring spec legacy test runner](https://github.com/mongodb/specifications/blob/09ee1ebc481f1502e3246971a9419e484d736207/source/command-monitoring/tests/README.rst). + events using the same logic as the + [Command Monitoring spec legacy test runner](../../command-logging-and-monitoring/tests/README.md). 9. For each element in `outcome`: - - If `name` is "collection", create a new MongoClient *without encryption* and verify that the test collection - contains exactly the documents in the `data` array. Ensure this find reads the latest data by using **primary read - preference** with **local read concern** even when the MongoClient is configured with another read preference or - read concern. + - If `name` is "collection", create a new MongoClient *without encryption* and verify that the test collection + contains exactly the documents in the `data` array. Ensure this find reads the latest data by using **primary + read preference** with **local read concern** even when the MongoClient is configured with another read + preference or read concern. The spec test MUST be run with *and* without auth. ## Using `crypt_shared` -On platforms where [crypt_shared](../client-side-encryption.rst#crypt_shared) is available, drivers should prefer to -test with the `crypt_shared` library instead of spawning mongocryptd. +On platforms where [crypt_shared](../client-side-encryption.md#crypt_shared) is available, drivers should prefer to test +with the `crypt_shared` library instead of spawning mongocryptd. -[crypt_shared](../client-side-encryption.rst#crypt_shared) is released alongside the server. -[crypt_shared](../client-side-encryption.rst#crypt_shared) is only available in versions 6.0 and above. +[crypt_shared](../client-side-encryption.md#crypt_shared) is released alongside the server. +[crypt_shared](../client-side-encryption.md#crypt_shared) is only available in versions 6.0 and above. mongocryptd is released alongside the server. mongocryptd is available in versions 4.2 and above. Drivers MUST run all tests with mongocryptd on at least one platform for all tested server versions. -Drivers MUST run all tests with [crypt_shared](../client-side-encryption.rst#crypt_shared) on at least one platform for +Drivers MUST run all tests with [crypt_shared](../client-side-encryption.md#crypt_shared) on at least one platform for all tested server versions. For server versions \< 6.0, drivers MUST test with the latest major release of -[crypt_shared](../client-side-encryption.rst#crypt_shared). Using the latest major release of -[crypt_shared](../client-side-encryption.rst#crypt_shared) is supported with older server versions. +[crypt_shared](../client-side-encryption.md#crypt_shared). Using the latest major release of +[crypt_shared](../client-side-encryption.md#crypt_shared) is supported with older server versions. Note that some tests assert on mongocryptd-related behaviors (e.g. the `mongocryptdBypassSpawn` test). -Drivers under test should load the [crypt_shared](../client-side-encryption.rst#crypt_shared) library using either the +Drivers under test should load the [crypt_shared](../client-side-encryption.md#crypt_shared) library using either the `cryptSharedLibPath` public API option (as part of the AutoEncryption `extraOptions`), or by setting a special search path instead. -Some tests will require *not* using [crypt_shared](../client-side-encryption.rst#crypt_shared). For such tests, one +Some tests will require *not* using [crypt_shared](../client-side-encryption.md#crypt_shared). For such tests, one should ensure that `crypt_shared` will not be loaded. Refer to the client-side-encryption documentation for information on "disabling" `crypt_shared` and setting library search paths. > [!NOTE] -> The [crypt_shared](../client-side-encryption.rst#crypt_shared) dynamic library can be obtained using the +> The [crypt_shared](../client-side-encryption.md#crypt_shared) dynamic library can be obtained using the > [mongodl](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/mongodl.py) Python script > from [drivers-evergreen-tools](https://github.com/mongodb-labs/drivers-evergreen-tools/): > @@ -346,7 +348,7 @@ Tests for the ClientEncryption type are not included as part of the YAML tests. In the prose tests LOCAL_MASTERKEY refers to the following base64: -```javascript +```text Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk ``` @@ -356,160 +358,166 @@ command) with readConcern/writeConcern "majority". ### 1. Custom Key Material Test 1. Create a `MongoClient` object (referred to as `client`). + 2. Using `client`, drop the collection `keyvault.datakeys`. + 3. Create a `ClientEncryption` object (referred to as `client_encryption`) with `client` set as the `keyVaultClient`. + 4. Using `client_encryption`, create a data key with a `local` KMS provider and the following custom key material (given - as base64): + as base64): -```javascript -xPTAjBRG5JiPm+d3fj6XLi2q5DMXUS/f1f+SMAlhhwkhDRL0kr8r9GDLIGTAGlvC+HVjSIgdL+RKwZCvpXSyxTICWSXTUYsWYPyu3IoHbuBZdmw2faM3WhcRIgbMReU5 -``` + ```text + xPTAjBRG5JiPm+d3fj6XLi2q5DMXUS/f1f+SMAlhhwkhDRL0kr8r9GDLIGTAGlvC+HVjSIgdL+RKwZCvpXSyxTICWSXTUYsWYPyu3IoHbuBZdmw2faM3WhcRIgbMReU5 + ``` -1. Find the resulting key document in `keyvault.datakeys`, save a copy of the key document, then remove the key document - from the collection. -2. Replace the `_id` field in the copied key document with a UUID with base64 value `AAAAAAAAAAAAAAAAAAAAAA==` (16 bytes - all equal to `0x00`) and insert the modified key document into `keyvault.datakeys` with majority write concern. -3. Using `client_encryption`, encrypt the string `"test"` with the modified data key using the - `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic` algorithm and assert the resulting value is equal to the following - (given as base64): +5. Find the resulting key document in `keyvault.datakeys`, save a copy of the key document, then remove the key document + from the collection. -```javascript -AQAAAAAAAAAAAAAAAAAAAAACz0ZOLuuhEYi807ZXTdhbqhLaS2/t9wLifJnnNYwiw79d75QYIZ6M/aYC1h9nCzCjZ7pGUpAuNnkUhnIXM3PjrA== -``` +6. Replace the `_id` field in the copied key document with a UUID with base64 value `AAAAAAAAAAAAAAAAAAAAAA==` (16 bytes + all equal to `0x00`) and insert the modified key document into `keyvault.datakeys` with majority write concern. + +7. Using `client_encryption`, encrypt the string `"test"` with the modified data key using the + `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic` algorithm and assert the resulting value is equal to the following + (given as base64): + + ```text + AQAAAAAAAAAAAAAAAAAAAAACz0ZOLuuhEYi807ZXTdhbqhLaS2/t9wLifJnnNYwiw79d75QYIZ6M/aYC1h9nCzCjZ7pGUpAuNnkUhnIXM3PjrA== + ``` ### 2. Data Key and Double Encryption First, perform the setup. 1. Create a MongoClient without encryption enabled (referred to as `client`). Enable command monitoring to listen for - command_started events. + command_started events. 2. Using `client`, drop the collections `keyvault.datakeys` and `db.coll`. 3. Create the following: - - A MongoClient configured with auto encryption (referred to as `client_encrypted`) - - A `ClientEncryption` object (referred to as `client_encryption`) + - A MongoClient configured with auto encryption (referred to as `client_encrypted`) + - A `ClientEncryption` object (referred to as `client_encryption`) - Configure both objects with the following KMS providers: + Configure both objects with the following KMS providers: - ```javascript - { - "aws": { - "accessKeyId": , - "secretAccessKey": - }, - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - }, - "gcp": { - "email": , - "privateKey": , - } - "local": { "key": }, - "kmip": { "endpoint": "localhost:5698" } - } - ``` - - Configure KMIP TLS connections to use the following options: - - - `tlsCAFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). - This MAY be configured system-wide. - - `tlsCertificateKeyFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). - - The method of passing TLS options for KMIP TLS connections is driver dependent. - - Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. - - Configure the `MongoClient` with the following `schema_map`: - - ```javascript - { - "db.coll": { - "bsonType": "object", - "properties": { - "encrypted_placeholder": { - "encrypt": { - "keyId": "/placeholder", - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" - } - } + ```javascript + { + "aws": { + "accessKeyId": , + "secretAccessKey": + }, + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + }, + "gcp": { + "email": , + "privateKey": , } - } - } - ``` + "local": { "key": }, + "kmip": { "endpoint": "localhost:5698" } + } + ``` - Configure `client_encryption` with the `keyVaultClient` of the previously created `client`. + Configure KMIP TLS connections to use the following options: + + - `tlsCAFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). + This MAY be configured system-wide. + - `tlsCertificateKeyFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + + + The method of passing TLS options for KMIP TLS connections is driver dependent. + + Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. + + Configure the `MongoClient` with the following `schema_map`: + + ```javascript + { + "db.coll": { + "bsonType": "object", + "properties": { + "encrypted_placeholder": { + "encrypt": { + "keyId": "/placeholder", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + } + } + } + } + ``` + + Configure `client_encryption` with the `keyVaultClient` of the previously created `client`. For each KMS provider (`aws`, `azure`, `gcp`, `local`, and `kmip`), referred to as `provider_name`, run the following test. 1. Call `client_encryption.createDataKey()`. - - Set keyAltNames to `["_altname"]`. + - Set keyAltNames to `["_altname"]`. - - Set the masterKey document based on `provider_name`. + - Set the masterKey document based on `provider_name`. - For "aws": + For "aws": - ```javascript - { - region: "us-east-1", - key: "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0" - } - ``` + ```javascript + { + region: "us-east-1", + key: "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0" + } + ``` - For "azure": + For "azure": - ```javascript - { - "keyVaultEndpoint": "key-vault-csfle.vault.azure.net", - "keyName": "key-name-csfle" - } - ``` + ```javascript + { + "keyVaultEndpoint": "key-vault-csfle.vault.azure.net", + "keyName": "key-name-csfle" + } + ``` - For "gcp": + For "gcp": - ```javascript - { - "projectId": "devprod-drivers", - "location": "global", - "keyRing": "key-ring-csfle", - "keyName": "key-name-csfle" - } - ``` + ```javascript + { + "projectId": "devprod-drivers", + "location": "global", + "keyRing": "key-ring-csfle", + "keyName": "key-name-csfle" + } + ``` - For "kmip": + For "kmip": - ```javascript - {} - ``` + ```javascript + {} + ``` - For "local", do not set a masterKey document. + For "local", do not set a masterKey document. - - Expect a BSON binary with subtype 4 to be returned, referred to as `datakey_id`. + - Expect a BSON binary with subtype 4 to be returned, referred to as `datakey_id`. - - Use `client` to run a `find` on `keyvault.datakeys` by querying with the `_id` set to the `datakey_id`. + - Use `client` to run a `find` on `keyvault.datakeys` by querying with the `_id` set to the `datakey_id`. - - Expect that exactly one document is returned with the "masterKey.provider" equal to `provider_name`. + - Expect that exactly one document is returned with the "masterKey.provider" equal to `provider_name`. - - Check that `client` captured a command_started event for the `insert` command containing a majority writeConcern. -2. Call `client_encryption.encrypt()` with the value "hello \", the algorithm - `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`, and the `key_id` of `datakey_id`. - - Expect the return value to be a BSON binary subtype 6, referred to as `encrypted`. - - Use `client_encrypted` to insert `{ _id: "", "value": }` into `db.coll`. - - Use `client_encrypted` to run a find querying with `_id` of "\" and expect `value` to be "hello - \". -3. Call `client_encryption.encrypt()` with the value "hello \", the algorithm - `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`, and the `key_alt_name` of `_altname`. - - Expect the return value to be a BSON binary subtype 6. Expect the value to exactly match the value of `encrypted`. + - Check that `client` captured a command_started event for the `insert` command containing a majority writeConcern. +2. Call `client_encryption.encrypt()` with the value `"hello "`, the algorithm + `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`, and the `key_id` of `datakey_id`. + - Expect the return value to be a BSON binary subtype 6, referred to as `encrypted`. + - Use `client_encrypted` to insert `{ _id: "", "value": }` into `db.coll`. + - Use `client_encrypted` to run a find querying with `_id` of `""` and expect `value` to be + `"hello "`. +3. Call `client_encryption.encrypt()` with the value `"hello "`, the algorithm + `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`, and the `key_alt_name` of `_altname`. + - Expect the return value to be a BSON binary subtype 6. Expect the value to exactly match the value of `encrypted`. 4. Test explicit encrypting an auto encrypted field. - - Use `client_encrypted` to attempt to insert `{ "encrypted_placeholder": }` - - Expect an exception to be thrown, since this is an attempt to auto encrypt an already encrypted value. + - Use `client_encrypted` to attempt to insert `{ "encrypted_placeholder": }` + - Expect an exception to be thrown, since this is an attempt to auto encrypt an already encrypted value. ### 3. External Key Vault Test @@ -518,35 +526,35 @@ Run the following tests twice, parameterized by a boolean `withExternalKeyVault` 1. Create a MongoClient without encryption enabled (referred to as `client`). 2. Using `client`, drop the collections `keyvault.datakeys` and `db.coll`. Insert the document - [external/external-key.json](../external/external-key.json) into `keyvault.datakeys`. + [external/external-key.json](../external/external-key.json) into `keyvault.datakeys`. 3. Create the following: - - A MongoClient configured with auto encryption (referred to as `client_encrypted`) - - A `ClientEncryption` object (referred to as `client_encryption`) + - A MongoClient configured with auto encryption (referred to as `client_encrypted`) + - A `ClientEncryption` object (referred to as `client_encryption`) - Configure both objects with the `local` KMS providers as follows: + Configure both objects with the `local` KMS providers as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. + Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. - Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for - `db.coll` by setting a schema map like: `{ "db.coll": }` + Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for + `db.coll` by setting a schema map like: `{ "db.coll": }` - If `withExternalKeyVault == true`, configure both objects with an external key vault client. The external client MUST - connect to the same MongoDB cluster that is being tested against, except it MUST use the username `fake-user` and - password `fake-pwd`. + If `withExternalKeyVault == true`, configure both objects with an external key vault client. The external client MUST + connect to the same MongoDB cluster that is being tested against, except it MUST use the username `fake-user` and + password `fake-pwd`. 4. Use `client_encrypted` to insert the document `{"encrypted": "test"}` into `db.coll`. If - `withExternalKeyVault == true`, expect an authentication exception to be thrown. Otherwise, expect the insert to - succeed. + `withExternalKeyVault == true`, expect an authentication exception to be thrown. Otherwise, expect the insert to + succeed. 5. Use `client_encryption` to explicitly encrypt the string `"test"` with key ID `LOCALAAAAAAAAAAAAAAAAA==` and - deterministic algorithm. If `withExternalKeyVault == true`, expect an authentication exception to be thrown. - Otherwise, expect the insert to succeed. + deterministic algorithm. If `withExternalKeyVault == true`, expect an authentication exception to be thrown. + Otherwise, expect the insert to succeed. ### 4. BSON Size Limits and Batch Splitting @@ -555,61 +563,61 @@ First, perform the setup. 1. Create a MongoClient without encryption enabled (referred to as `client`). 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema - [limits/limits-schema.json](../limits/limits-schema.json). + [limits/limits-schema.json](../limits/limits-schema.json). 3. Using `client`, drop the collection `keyvault.datakeys`. Insert the document - [limits/limits-key.json](../limits/limits-key.json) + [limits/limits-key.json](../limits/limits-key.json) 4. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure with the `local` KMS provider as follows: + Configure with the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. Using `client_encrypted` perform the following operations: 1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }`. - Expect this to succeed since this is still under the `maxBsonObjectSize` limit. + Expect this to succeed since this is still under the `maxBsonObjectSize` limit. 2. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` Note: - limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. + `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` Note: + limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. - Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before - auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT - exceed the 16 MiB limit. + Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before + auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT + exceed the 16 MiB limit. 3. Bulk insert the following: - - `{ "_id": "over_2mib_1", "unencrypted": }` - - `{ "_id": "over_2mib_2", "unencrypted": }` + - `{ "_id": "over_2mib_1", "unencrypted": }` + - `{ "_id": "over_2mib_2", "unencrypted": }` - Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using - [command monitoring](https://github.com/mongodb/specifications/tree/master/source/command-logging-and-monitoring/command-logging-and-monitoring.rst). + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). 4. Bulk insert the following: - - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib_1", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` - - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib_2", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` + - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_1", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` + - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_2", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` - Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using - [command logging and monitoring](https://github.com/mongodb/specifications/tree/master/source/command-logging-and-monitoring/command-logging-and-monitoring.rst). + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). 5. Insert `{ "_id": "under_16mib", "unencrypted": `. - Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. + Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. 6. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` + `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` - Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. + Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. Optionally, if it is possible to mock the maxWriteBatchSize (i.e. the maximum number of documents in a batch) test that setting maxWriteBatchSize=1 and inserting the two documents `{ "_id": "a" }, { "_id": "b" }` with `client_encrypted` @@ -620,20 +628,20 @@ splits the operation into two inserts. 1. Create a MongoClient without encryption enabled (referred to as `client`). 2. Using `client`, drop and create a view named `db.view` with an empty pipeline. E.g. using the command - `{ "create": "view", "viewOn": "coll" }`. + `{ "create": "view", "viewOn": "coll" }`. 3. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure with the `local` KMS provider as follows: + Configure with the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. 4. Using `client_encrypted`, attempt to insert a document into `db.view`. Expect an exception to be thrown containing - the message: "cannot auto encrypt a view". + the message: "cannot auto encrypt a view". ### 6. Corpus Test @@ -643,108 +651,109 @@ binary subtype 4 (or standard UUID), which MUST be decoded and encoded as subtyp 1. Create a MongoClient without encryption enabled (referred to as `client`). 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema - [corpus/corpus-schema.json](../corpus/corpus-schema.json). + [corpus/corpus-schema.json](../corpus/corpus-schema.json). 3. Using `client`, drop the collection `keyvault.datakeys`. Insert the documents - [corpus/corpus-key-local.json](../corpus/corpus-key-local.json), - [corpus/corpus-key-aws.json](../corpus/corpus-key-aws.json), - [corpus/corpus-key-azure.json](../corpus/corpus-key-azure.json), - [corpus/corpus-key-gcp.json](../corpus/corpus-key-gcp.json), and - [corpus/corpus-key-kmip.json](../corpus/corpus-key-kmip.json). + [corpus/corpus-key-local.json](../corpus/corpus-key-local.json), + [corpus/corpus-key-aws.json](../corpus/corpus-key-aws.json), + [corpus/corpus-key-azure.json](../corpus/corpus-key-azure.json), + [corpus/corpus-key-gcp.json](../corpus/corpus-key-gcp.json), and + [corpus/corpus-key-kmip.json](../corpus/corpus-key-kmip.json). 4. Create the following: - - A MongoClient configured with auto encryption (referred to as `client_encrypted`) - - A `ClientEncryption` object (referred to as `client_encryption`) + - A MongoClient configured with auto encryption (referred to as `client_encrypted`) + - A `ClientEncryption` object (referred to as `client_encryption`) - Configure both objects with `aws`, `azure`, `gcp`, `local`, and `kmip` KMS providers as follows: + Configure both objects with `aws`, `azure`, `gcp`, `local`, and `kmip` KMS providers as follows: - ```javascript - { - "aws": { }, - "azure": { }, - "gcp": { }, - "local": { "key": }, - "kmip": { "endpoint": "localhost:5698" } } - } - ``` + ```javascript + { + "aws": { }, + "azure": { }, + "gcp": { }, + "local": { "key": }, + "kmip": { "endpoint": "localhost:5698" } + } + ``` - Configure KMIP TLS connections to use the following options: + Configure KMIP TLS connections to use the following options: - - `tlsCAFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). - This MAY be configured system-wide. - - `tlsCertificateKeyFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + - `tlsCAFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). + This MAY be configured system-wide. + - `tlsCertificateKeyFile` (or equivalent) set to + [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + - The method of passing TLS options for KMIP TLS connections is driver dependent. + The method of passing TLS options for KMIP TLS connections is driver dependent. - Where LOCAL_MASTERKEY is the following base64: + Where LOCAL_MASTERKEY is the following base64: - ```javascript - Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - ``` + ```text + Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + ``` - Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. + Configure both objects with `keyVaultNamespace` set to `keyvault.datakeys`. 5. Load [corpus/corpus.json](../corpus/corpus.json) to a variable named `corpus`. The corpus contains subdocuments with - the following fields: - - - `kms` is `aws`, `azure`, `gcp`, `local`, or `kmip` - - `type` is a BSON type string - [names coming from here](https://www.mongodb.com/docs/manual/reference/operator/query/type/)) - - `algo` is either `rand` or `det` for random or deterministic encryption - - `method` is either `auto`, for automatic encryption or `explicit` for explicit encryption - - `identifier` is either `id` or `altname` for the key identifier - - `allowed` is a boolean indicating whether the encryption for the given parameters is permitted. - - `value` is the value to be tested. - - Create a new BSON document, named `corpus_copied`. Iterate over each field of `corpus`. - - - If the field name is `_id`, `altname_aws`, `altname_local`, `altname_azure`, `altname_gcp`, or `altname_kmip` copy - the field to `corpus_copied`. - - - If `method` is `auto`, copy the field to `corpus_copied`. - - - If `method` is `explicit`, use `client_encryption` to explicitly encrypt the value. - - - Encrypt with the algorithm described by `algo`. - - If `identifier` is `id` - - If `kms` is `local` set the key_id to the UUID with base64 value `LOCALAAAAAAAAAAAAAAAAA==`. - - If `kms` is `aws` set the key_id to the UUID with base64 value `AWSAAAAAAAAAAAAAAAAAAA==`. - - If `kms` is `azure` set the key_id to the UUID with base64 value `AZUREAAAAAAAAAAAAAAAAA==`. - - If `kms` is `gcp` set the key_id to the UUID with base64 value `GCPAAAAAAAAAAAAAAAAAAA==`. - - If `kms` is `kmip` set the key_id to the UUID with base64 value `KMIPAAAAAAAAAAAAAAAAAA==`. - - If `identifier` is `altname` - - If `kms` is `local` set the key_alt_name to "local". - - If `kms` is `aws` set the key_alt_name to "aws". - - If `kms` is `azure` set the key_alt_name to "azure". - - If `kms` is `gcp` set the key_alt_name to "gcp". - - If `kms` is `kmip` set the key_alt_name to "kmip". - - If `allowed` is true, copy the field and encrypted value to `corpus_copied`. If `allowed` is false. verify that an - exception is thrown. Copy the unencrypted value to to `corpus_copied`. + the following fields: + + - `kms` is `aws`, `azure`, `gcp`, `local`, or `kmip` + - `type` is a BSON type string + [names coming from here](https://www.mongodb.com/docs/manual/reference/operator/query/type/)) + - `algo` is either `rand` or `det` for random or deterministic encryption + - `method` is either `auto`, for automatic encryption or `explicit` for explicit encryption + - `identifier` is either `id` or `altname` for the key identifier + - `allowed` is a boolean indicating whether the encryption for the given parameters is permitted. + - `value` is the value to be tested. + + Create a new BSON document, named `corpus_copied`. Iterate over each field of `corpus`. + + - If the field name is `_id`, `altname_aws`, `altname_local`, `altname_azure`, `altname_gcp`, or `altname_kmip` copy + the field to `corpus_copied`. + + - If `method` is `auto`, copy the field to `corpus_copied`. + + - If `method` is `explicit`, use `client_encryption` to explicitly encrypt the value. + + - Encrypt with the algorithm described by `algo`. + - If `identifier` is `id` + - If `kms` is `local` set the key_id to the UUID with base64 value `LOCALAAAAAAAAAAAAAAAAA==`. + - If `kms` is `aws` set the key_id to the UUID with base64 value `AWSAAAAAAAAAAAAAAAAAAA==`. + - If `kms` is `azure` set the key_id to the UUID with base64 value `AZUREAAAAAAAAAAAAAAAAA==`. + - If `kms` is `gcp` set the key_id to the UUID with base64 value `GCPAAAAAAAAAAAAAAAAAAA==`. + - If `kms` is `kmip` set the key_id to the UUID with base64 value `KMIPAAAAAAAAAAAAAAAAAA==`. + - If `identifier` is `altname` + - If `kms` is `local` set the key_alt_name to "local". + - If `kms` is `aws` set the key_alt_name to "aws". + - If `kms` is `azure` set the key_alt_name to "azure". + - If `kms` is `gcp` set the key_alt_name to "gcp". + - If `kms` is `kmip` set the key_alt_name to "kmip". + + If `allowed` is true, copy the field and encrypted value to `corpus_copied`. If `allowed` is false. verify that an + exception is thrown. Copy the unencrypted value to to `corpus_copied`. 6. Using `client_encrypted`, insert `corpus_copied` into `db.coll`. 7. Using `client_encrypted`, find the inserted document from `db.coll` to a variable named `corpus_decrypted`. Since it - should have been automatically decrypted, assert the document exactly matches `corpus`. + should have been automatically decrypted, assert the document exactly matches `corpus`. 8. Load [corpus/corpus_encrypted.json](../corpus/corpus-encrypted.json) to a variable named `corpus_encrypted_expected`. - Using `client` find the inserted document from `db.coll` to a variable named `corpus_encrypted_actual`. + Using `client` find the inserted document from `db.coll` to a variable named `corpus_encrypted_actual`. - Iterate over each field of `corpus_encrypted_expected` and check the following: + Iterate over each field of `corpus_encrypted_expected` and check the following: - - If the `algo` is `det`, that the value equals the value of the corresponding field in `corpus_encrypted_actual`. - - If the `algo` is `rand` and `allowed` is true, that the value does not equal the value of the corresponding field - in `corpus_encrypted_actual`. - - If `allowed` is true, decrypt the value with `client_encryption`. Decrypt the value of the corresponding field of - `corpus_encrypted` and validate that they are both equal. - - If `allowed` is false, validate the value exactly equals the value of the corresponding field of `corpus` (neither - was encrypted). + - If the `algo` is `det`, that the value equals the value of the corresponding field in `corpus_encrypted_actual`. + - If the `algo` is `rand` and `allowed` is true, that the value does not equal the value of the corresponding field + in `corpus_encrypted_actual`. + - If `allowed` is true, decrypt the value with `client_encryption`. Decrypt the value of the corresponding field of + `corpus_encrypted` and validate that they are both equal. + - If `allowed` is false, validate the value exactly equals the value of the corresponding field of `corpus` (neither + was encrypted). 9. Repeat steps 1-8 with a local JSON schema. I.e. amend step 4 to configure the schema on `client_encrypted` with the - `schema_map` option. + `schema_map` option. ### 7. Custom Endpoint Test @@ -810,16 +819,16 @@ Configure with KMS providers as follows: Configure KMIP TLS connections to use the following options: - `tlsCAFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). - This MAY be configured system-wide. + [drivers-evergreen-tools/.evergreen/x509gen/ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). + This MAY be configured system-wide. - `tlsCertificateKeyFile` (or equivalent) set to - [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). + [drivers-evergreen-tools/.evergreen/x509gen/client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem). The method of passing TLS options for KMIP TLS connections is driver dependent. #### Test cases -01. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +1. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -831,7 +840,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to succeed. Use the returned UUID of the key to explicitly encrypt and decrypt the string "test" to validate it works. -02. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +2. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -844,7 +853,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to succeed. Use the returned UUID of the key to explicitly encrypt and decrypt the string "test" to validate it works. -03. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +3. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -857,7 +866,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to succeed. Use the returned UUID of the key to explicitly encrypt and decrypt the string "test" to validate it works. -04. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +4. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -869,7 +878,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to fail with a socket connection error. -05. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +5. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -881,7 +890,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to fail with an exception. -06. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: +6. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: ```javascript { @@ -893,7 +902,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Expect this to fail with a network exception indicating failure to resolve "doesnotexist.invalid". -07. Call `client_encryption.createDataKey()` with "azure" as the provider and the following masterKey: +7. Call `client_encryption.createDataKey()` with "azure" as the provider and the following masterKey: ```javascript { @@ -908,7 +917,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Call `client_encryption_invalid.createDataKey()` with the same masterKey. Expect this to fail with a network exception indicating failure to resolve "doesnotexist.invalid". -08. Call `client_encryption.createDataKey()` with "gcp" as the provider and the following masterKey: +8. Call `client_encryption.createDataKey()` with "gcp" as the provider and the following masterKey: ```javascript { @@ -926,7 +935,7 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. Call `client_encryption_invalid.createDataKey()` with the same masterKey. Expect this to fail with a network exception indicating failure to resolve "doesnotexist.invalid". -09. Call `client_encryption.createDataKey()` with "gcp" as the provider and the following masterKey: +9. Call `client_encryption.createDataKey()` with "gcp" as the provider and the following masterKey: ```javascript { @@ -986,48 +995,48 @@ The method of passing TLS options for KMIP TLS connections is driver dependent. #### Via loading shared library -The following tests that loading [crypt_shared](../client-side-encryption.rst#crypt_shared) bypasses spawning +The following tests that loading [crypt_shared](../client-side-encryption.md#crypt_shared) bypasses spawning mongocryptd. > [!NOTE] -> IMPORTANT: This test requires the [crypt_shared](../client-side-encryption.rst#crypt_shared) library be loaded. If the -> [crypt_shared](../client-side-encryption.rst#crypt_shared) library is not available, skip the test. +> IMPORTANT: This test requires the [crypt_shared](../client-side-encryption.md#crypt_shared) library be loaded. If the +> [crypt_shared](../client-side-encryption.md#crypt_shared) library is not available, skip the test. 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure the required options. Use the `local` KMS provider as follows: + Configure the required options. Use the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. - Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for - `db.coll` by setting a schema map like: `{ "db.coll": }` + Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for + `db.coll` by setting a schema map like: `{ "db.coll": }` - Configure the following `extraOptions`: + Configure the following `extraOptions`: - ```javascript - { - "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000", - "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"], - "cryptSharedLibPath": "", - "cryptSharedLibRequired": true - } - ``` + ```javascript + { + "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000", + "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"], + "cryptSharedLibPath": "", + "cryptSharedLibRequired": true + } + ``` - Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port - that should be free. + Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port + that should be free. 2. Use `client_encrypted` to insert the document `{"unencrypted": "test"}` into `db.coll`. Expect this to succeed. 3. Validate that mongocryptd was not spawned. Create a MongoClient to localhost:27021 (or whatever was passed via - `--port`) with serverSelectionTimeoutMS=1000. Run a handshake command and ensure it fails with a server selection - timeout. + `--port`) with serverSelectionTimeoutMS=1000. Run a handshake command and ensure it fails with a server selection + timeout. > [!NOTE] -> IMPORTANT: If [crypt_shared](../client-side-encryption.rst#crypt_shared) is visible to the operating system's library +> IMPORTANT: If [crypt_shared](../client-side-encryption.md#crypt_shared) is visible to the operating system's library > search mechanism, the expected server error generated by the `Via mongocryptdBypassSpawn`, `Via bypassAutoEncryption`, > `Via bypassQueryAnalysis` tests will not appear because libmongocrypt will load the `crypt_shared` library instead of > consulting mongocryptd. For the following tests, it is required that libmongocrypt *not* load `crypt_shared`. Refer to @@ -1040,37 +1049,37 @@ mongocryptd. The following tests that setting `mongocryptdBypassSpawn=true` really does bypass spawning mongocryptd. 1. Insert the document [external/external-key.json](../external/external-key.json) into `keyvault.datakeys` with - majority write concern. This step is not required to run this test, and drivers MAY skip it. But if the driver - misbehaves, then not having the encryption fully set up may complicate the process of figuring out what is wrong. + majority write concern. This step is not required to run this test, and drivers MAY skip it. But if the driver + misbehaves, then not having the encryption fully set up may complicate the process of figuring out what is wrong. 2. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure the required options. Use the `local` KMS provider as follows: + Configure the required options. Use the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. - Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for - `db.coll` by setting a schema map like: `{ "db.coll": }` + Configure `client_encrypted` to use the schema [external/external-schema.json](../external/external-schema.json) for + `db.coll` by setting a schema map like: `{ "db.coll": }` - Configure the following `extraOptions`: + Configure the following `extraOptions`: - ```javascript - { - "mongocryptdBypassSpawn": true - "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000", - "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"] - } - ``` + ```javascript + { + "mongocryptdBypassSpawn": true + "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000", + "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"] + } + ``` - Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port - that should be free. + Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port + that should be free. 3. Use `client_encrypted` to insert the document `{"encrypted": "test"}` into `db.coll`. Expect a server selection error - propagated from the internal MongoClient failing to connect to mongocryptd on port 27021. + propagated from the internal MongoClient failing to connect to mongocryptd on port 27021. #### Via bypassAutoEncryption @@ -1078,32 +1087,32 @@ The following tests that setting `bypassAutoEncryption=true` really does bypass 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure the required options. Use the `local` KMS provider as follows: + Configure the required options. Use the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. - Configure with `bypassAutoEncryption=true`. + Configure with `bypassAutoEncryption=true`. - Configure the following `extraOptions`: + Configure the following `extraOptions`: - ```javascript - { - "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"] - } - ``` + ```javascript + { + "mongocryptdSpawnArgs": [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=27021"] + } + ``` - Drivers MAY pass a different value to `--port` if they expect their testing infrastructure to be using port 27021. - Pass a port that should be free. + Drivers MAY pass a different value to `--port` if they expect their testing infrastructure to be using port 27021. + Pass a port that should be free. 2. Use `client_encrypted` to insert the document `{"unencrypted": "test"}` into `db.coll`. Expect this to succeed. 3. Validate that mongocryptd was not spawned. Create a MongoClient to localhost:27021 (or whatever was passed via - `--port`) with serverSelectionTimeoutMS=1000. Run a handshake command and ensure it fails with a server selection - timeout. + `--port`) with serverSelectionTimeoutMS=1000. Run a handshake command and ensure it fails with a server selection + timeout. #### Via bypassQueryAnalysis @@ -1154,23 +1163,23 @@ Each test must assert the number of unique `MongoClient` objects created. This c - Create a `MongoClient` named `client_encrypted` configured as follows: - - Set `AutoEncryptionOpts`: + - Set `AutoEncryptionOpts`: - - `keyVaultNamespace="keyvault.datakeys"` - - `kmsProviders`=`{ "local": { "key": } }` - - Append `TestCase.AutoEncryptionOpts` (defined below) + - `keyVaultNamespace="keyvault.datakeys"` + - `kmsProviders`=`{ "local": { "key": } }` + - Append `TestCase.AutoEncryptionOpts` (defined below) - - Capture command started events. + - Capture command started events. - - Set `maxPoolSize=TestCase.MaxPoolSize` + - Set `maxPoolSize=TestCase.MaxPoolSize` - If the testcase sets `AutoEncryptionOpts.bypassAutoEncryption=true`: - - Use `client_test` to insert `{ "_id": 0, "encrypted": }` into `db.coll`. + - Use `client_test` to insert `{ "_id": 0, "encrypted": }` into `db.coll`. - Otherwise: - - Use `client_encrypted` to insert `{ "_id": 0, "encrypted": "string0" }`. + - Use `client_encrypted` to insert `{ "_id": 0, "encrypted": "string0" }`. - Use `client_encrypted` to run a `findOne` operation on `db.coll`, with the filter `{ "_id": 0 }`. @@ -1186,16 +1195,16 @@ Each test must assert the number of unique `MongoClient` objects created. This c - AutoEncryptionOpts: - - bypassAutoEncryption=false - - keyVaultClient=unset + - bypassAutoEncryption=false + - keyVaultClient=unset - Expectations: - - Expect `client_encrypted` to have captured four `CommandStartedEvent`: - - a listCollections to "db". - - a find on "keyvault". - - an insert on "db". - - a find on "db" + - Expect `client_encrypted` to have captured four `CommandStartedEvent`: + - a listCollections to "db". + - a find on "keyvault". + - an insert on "db". + - a find on "db" - ExpectedNumberOfClients: 2 @@ -1205,20 +1214,20 @@ Each test must assert the number of unique `MongoClient` objects created. This c - AutoEncryptionOpts: - - bypassAutoEncryption=false - - keyVaultClient=client_keyvault + - bypassAutoEncryption=false + - keyVaultClient=client_keyvault - Expectations: - - Expect `client_encrypted` to have captured three `CommandStartedEvent`: + - Expect `client_encrypted` to have captured three `CommandStartedEvent`: - - a listCollections to "db". - - an insert on "db". - - a find on "db" + - a listCollections to "db". + - an insert on "db". + - a find on "db" - - Expect `client_keyvault` to have captured one `CommandStartedEvent`: + - Expect `client_keyvault` to have captured one `CommandStartedEvent`: - - a find on "keyvault". + - a find on "keyvault". - ExpectedNumberOfClients: 2 @@ -1228,14 +1237,14 @@ Each test must assert the number of unique `MongoClient` objects created. This c - AutoEncryptionOpts: - - bypassAutoEncryption=true - - keyVaultClient=unset + - bypassAutoEncryption=true + - keyVaultClient=unset - Expectations: - - Expect `client_encrypted` to have captured three `CommandStartedEvent`: - - a find on "db" - - a find on "keyvault". + - Expect `client_encrypted` to have captured three `CommandStartedEvent`: + - a find on "db" + - a find on "keyvault". - ExpectedNumberOfClients: 2 @@ -1245,18 +1254,18 @@ Each test must assert the number of unique `MongoClient` objects created. This c - AutoEncryptionOpts: - - bypassAutoEncryption=true - - keyVaultClient=client_keyvault + - bypassAutoEncryption=true + - keyVaultClient=client_keyvault - Expectations: - - Expect `client_encrypted` to have captured two `CommandStartedEvent`: + - Expect `client_encrypted` to have captured two `CommandStartedEvent`: - - a find on "db" + - a find on "db" - - Expect `client_keyvault` to have captured one `CommandStartedEvent`: + - Expect `client_keyvault` to have captured one `CommandStartedEvent`: - - a find on "keyvault". + - a find on "keyvault". - ExpectedNumberOfClients: 1 @@ -1268,17 +1277,17 @@ Drivers that do not support an unlimited maximum pool size MUST skip this test. - AutoEncryptionOpts: - - bypassAutoEncryption=false - - keyVaultClient=unset + - bypassAutoEncryption=false + - keyVaultClient=unset - Expectations: - - Expect `client_encrypted` to have captured five `CommandStartedEvent`: - - a listCollections to "db". - - a listCollections to "keyvault". - - a find on "keyvault". - - an insert on "db". - - a find on "db" + - Expect `client_encrypted` to have captured five `CommandStartedEvent`: + - a listCollections to "db". + - a listCollections to "keyvault". + - a find on "keyvault". + - an insert on "db". + - a find on "db" - ExpectedNumberOfClients: 1 @@ -1290,20 +1299,20 @@ Drivers that do not support an unlimited maximum pool size MUST skip this test. - AutoEncryptionOpts: - - bypassAutoEncryption=false - - keyVaultClient=client_keyvault + - bypassAutoEncryption=false + - keyVaultClient=client_keyvault - Expectations: - - Expect `client_encrypted` to have captured three `CommandStartedEvent`: + - Expect `client_encrypted` to have captured three `CommandStartedEvent`: - - a listCollections to "db". - - an insert on "db". - - a find on "db" + - a listCollections to "db". + - an insert on "db". + - a find on "db" - - Expect `client_keyvault` to have captured one `CommandStartedEvent`: + - Expect `client_keyvault` to have captured one `CommandStartedEvent`: - - a find on "keyvault". + - a find on "keyvault". - ExpectedNumberOfClients: 1 @@ -1315,14 +1324,14 @@ Drivers that do not support an unlimited maximum pool size MUST skip this test. - AutoEncryptionOpts: - - bypassAutoEncryption=true - - keyVaultClient=unset + - bypassAutoEncryption=true + - keyVaultClient=unset - Expectations: - - Expect `client_encrypted` to have captured three `CommandStartedEvent`: - - a find on "db" - - a find on "keyvault". + - Expect `client_encrypted` to have captured three `CommandStartedEvent`: + - a find on "db" + - a find on "keyvault". - ExpectedNumberOfClients: 1 @@ -1334,18 +1343,18 @@ Drivers that do not support an unlimited maximum pool size MUST skip this test. - AutoEncryptionOpts: - - bypassAutoEncryption=true - - keyVaultClient=client_keyvault + - bypassAutoEncryption=true + - keyVaultClient=client_keyvault - Expectations: - - Expect `client_encrypted` to have captured two `CommandStartedEvent`: + - Expect `client_encrypted` to have captured two `CommandStartedEvent`: - - a find on "db" + - a find on "db" - - Expect `client_keyvault` to have captured one `CommandStartedEvent`: + - Expect `client_keyvault` to have captured one `CommandStartedEvent`: - - a find on "keyvault". + - a find on "keyvault". - ExpectedNumberOfClients: 1 @@ -1366,7 +1375,7 @@ cert file, and one on port 9001 with [wrong-host.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/wrong-host.pem) as a cert file, run the following commands from the `.evergreen/csfle` directory: -``` +```shell . ./activate_venv.sh python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 & python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 & @@ -1379,54 +1388,55 @@ For both tests, do the following: 1. Start a `mongod` process with **server version 4.2.0 or later**. 2. Create a `MongoClient` for key vault operations. 3. Create a `ClientEncryption` object (referred to as `client_encryption`) with `keyVaultNamespace` set to - `keyvault.datakeys`. + `keyvault.datakeys`. #### Invalid KMS Certificate 1. Start a mock KMS server on port 9000 with - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA file - and [expired.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/expired.pem) - as a cert file. + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA + file and + [expired.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/expired.pem) + as a cert file. 2. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: - ```javascript - { - "region": "us-east-1", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "endpoint": "127.0.0.1:9000", - } - ``` + ```javascript + { + "region": "us-east-1", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "endpoint": "127.0.0.1:9000", + } + ``` - Expect this to fail with an exception with a message referencing an expired certificate. This message will be - language dependent. In Python, this message is "certificate verify failed: certificate has expired". In Go, this - message is "certificate has expired or is not yet valid". If the language of implementation has a single, generic - error message for all certificate validation errors, drivers may inspect other fields of the error to verify its - meaning. + Expect this to fail with an exception with a message referencing an expired certificate. This message will be + language dependent. In Python, this message is "certificate verify failed: certificate has expired". In Go, this + message is "certificate has expired or is not yet valid". If the language of implementation has a single, generic + error message for all certificate validation errors, drivers may inspect other fields of the error to verify its + meaning. #### Invalid Hostname in KMS Certificate 1. Start a mock KMS server on port 9001 with - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA file - and - [wrong-host.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/wrong-host.pem) - as a cert file. + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA + file and + [wrong-host.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/wrong-host.pem) + as a cert file. 2. Call `client_encryption.createDataKey()` with "aws" as the provider and the following masterKey: - ```javascript - { - "region": "us-east-1", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "endpoint": "127.0.0.1:9001", - } - ``` + ```javascript + { + "region": "us-east-1", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "endpoint": "127.0.0.1:9001", + } + ``` - Expect this to fail with an exception with a message referencing an incorrect or unexpected host. This message will - be language dependent. In Python, this message is "certificate verify failed: IP address mismatch, certificate is not - valid for '127.0.0.1'". In Go, this message is "cannot validate certificate for 127.0.0.1 because it doesn't contain - any IP SANs". If the language of implementation has a single, generic error message for all certificate validation - errors, drivers may inspect other fields of the error to verify its meaning. + Expect this to fail with an exception with a message referencing an incorrect or unexpected host. This message will + be language dependent. In Python, this message is "certificate verify failed: IP address mismatch, certificate is + not valid for '127.0.0.1'". In Go, this message is "cannot validate certificate for 127.0.0.1 because it doesn't + contain any IP SANs". If the language of implementation has a single, generic error message for all certificate + validation errors, drivers may inspect other fields of the error to verify its meaning. ### 11. KMS TLS Options Tests @@ -1437,52 +1447,57 @@ Start a `mongod` process with **server version 4.2.0 or later**. Four mock KMS server processes must be running: 1. The mock - [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + - Run on port 9000 with - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA file - and [expired.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/expired.pem) - as a cert file. + Run on port 9000 with + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA + file and + [expired.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/expired.pem) + as a cert file. - Example: + Example: - ``` - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 - ``` + ```shell + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 + ``` 2. The mock - [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + - Run on port 9001 with - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA file - and - [wrong-host.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/wrong-host.pem) - as a cert file. + Run on port 9001 with + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA + file and + [wrong-host.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/wrong-host.pem) + as a cert file. - Example: + Example: - ``` - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 - ``` + ```shell + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 + ``` 3. The mock - [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + [KMS HTTP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_http_server.py). + - Run on port 9002 with - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA file - and [server.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/server.pem) - as a cert file. + Run on port 9002 with + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem) as a CA + file and + [server.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/server.pem) as + a cert file. - Run with the `--require_client_cert` option. + Run with the `--require_client_cert` option. - Example: + Example: - ``` - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 9002 --require_client_cert - ``` + ```shell + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 9002 --require_client_cert + ``` 4. The mock - [KMS KMIP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_kmip_server.py). + [KMS KMIP server](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/kms_kmip_server.py). Create the following `ClientEncryption` objects. @@ -1490,190 +1505,190 @@ Configure each with `keyVaultNamespace` set to `keyvault.datakeys`, and a defaul 1. Create a `ClientEncryption` object named `client_encryption_no_client_cert` with the following KMS providers: - ```javascript - { - "aws": { - "accessKeyId": , - "secretAccessKey": - }, - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9002" - }, - "gcp": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9002" - }, - "kmip" { - "endpoint": "127.0.0.1:5698" - } - } - ``` + ```javascript + { + "aws": { + "accessKeyId": , + "secretAccessKey": + }, + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9002" + }, + "gcp": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9002" + }, + "kmip" { + "endpoint": "127.0.0.1:5698" + } + } + ``` - Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: + Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. 2. Create a `ClientEncryption` object named `client_encryption_with_tls` with the following KMS providers: - ```javascript - { - "aws": { - "accessKeyId": , - "secretAccessKey": - }, - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9002" - }, - "gcp": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9002" - }, - "kmip" { - "endpoint": "127.0.0.1:5698" - } - } - ``` + ```javascript + { + "aws": { + "accessKeyId": , + "secretAccessKey": + }, + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9002" + }, + "gcp": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9002" + }, + "kmip" { + "endpoint": "127.0.0.1:5698" + } + } + ``` - Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: + Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. - - `tlsCertificateKeyFile` (or equivalent) set to - [client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem) + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. + - `tlsCertificateKeyFile` (or equivalent) set to + [client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem) 3. Create a `ClientEncryption` object named `client_encryption_expired` with the following KMS providers: - ```javascript - { - "aws": { - "accessKeyId": , - "secretAccessKey": - }, - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9000" - }, - "gcp": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9000" - }, - "kmip" { - "endpoint": "127.0.0.1:9000" - } - } - ``` + ```javascript + { + "aws": { + "accessKeyId": , + "secretAccessKey": + }, + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9000" + }, + "gcp": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9000" + }, + "kmip" { + "endpoint": "127.0.0.1:9000" + } + } + ``` - Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: + Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. 4. Create a `ClientEncryption` object named `client_encryption_invalid_hostname` with the following KMS providers: - ```javascript - { - "aws": { - "accessKeyId": , - "secretAccessKey": - }, - "azure": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9001" - }, - "gcp": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9001" - }, - "kmip" { - "endpoint": "127.0.0.1:9001" - } - } - ``` + ```javascript + { + "aws": { + "accessKeyId": , + "secretAccessKey": + }, + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9001" + }, + "gcp": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9001" + }, + "kmip" { + "endpoint": "127.0.0.1:9001" + } + } + ``` - Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: + Add TLS options for the `aws`, `azure`, `gcp`, and `kmip` providers to use the following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. 5. Create a `ClientEncryption` object named `client_encryption_with_names` with the following KMS providers: - ```javascript - { - "aws:no_client_cert": { - "accessKeyId": , - "secretAccessKey": - }, - "azure:no_client_cert": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9002" - }, - "gcp:no_client_cert": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9002" - }, - "kmip:no_client_cert": { - "endpoint": "127.0.0.1:5698" - }, - "aws:with_tls": { - "accessKeyId": , - "secretAccessKey": - }, - "azure:with_tls": { - "tenantId": , - "clientId": , - "clientSecret": , - "identityPlatformEndpoint": "127.0.0.1:9002" - }, - "gcp:with_tls": { - "email": , - "privateKey": , - "endpoint": "127.0.0.1:9002" - }, - "kmip:with_tls": { - "endpoint": "127.0.0.1:5698" - } - } - ``` + ```javascript + { + "aws:no_client_cert": { + "accessKeyId": , + "secretAccessKey": + }, + "azure:no_client_cert": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9002" + }, + "gcp:no_client_cert": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9002" + }, + "kmip:no_client_cert": { + "endpoint": "127.0.0.1:5698" + }, + "aws:with_tls": { + "accessKeyId": , + "secretAccessKey": + }, + "azure:with_tls": { + "tenantId": , + "clientId": , + "clientSecret": , + "identityPlatformEndpoint": "127.0.0.1:9002" + }, + "gcp:with_tls": { + "email": , + "privateKey": , + "endpoint": "127.0.0.1:9002" + }, + "kmip:with_tls": { + "endpoint": "127.0.0.1:5698" + } + } + ``` - Support for named KMS providers requires libmongocrypt 1.9.0. + Support for named KMS providers requires libmongocrypt 1.9.0. - Add TLS options for the `aws:no_client_cert`, `azure:no_client_cert`, `gcp:no_client_cert`, and `kmip:no_client_cert` - providers to use the following options: + Add TLS options for the `aws:no_client_cert`, `azure:no_client_cert`, `gcp:no_client_cert`, and `kmip:no_client_cert` + providers to use the following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. - Add TLS options for the `aws:with_tls`, `azure:with_tls`, `gcp:with_tls`, and `kmip:with_tls` providers to use the - following options: + Add TLS options for the `aws:with_tls`, `azure:with_tls`, `gcp:with_tls`, and `kmip:with_tls` providers to use the + following options: - - `tlsCAFile` (or equivalent) set to - [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY - be configured system-wide. - - `tlsCertificateKeyFile` (or equivalent) set to - [client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem) + - `tlsCAFile` (or equivalent) set to + [ca.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/ca.pem). This MAY + be configured system-wide. + - `tlsCertificateKeyFile` (or equivalent) set to + [client.pem](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/x509gen/client.pem) #### Case 1: AWS @@ -1907,7 +1922,7 @@ as `key1Document`. Read the `"_id"` field of `key1Document` as `key1ID`. Drop and create the collection `db.explicit_encryption` using `encryptedFields` as an option. See -[FLE 2 CreateCollection() and Collection.Drop()](../client-side-encryption.md#fle-2-createcollection-and-collection-drop). +[FLE 2 CreateCollection() and Collection.Drop()](../client-side-encryption.md#create-collection-helper). Drop and create the collection `keyvault.datakeys`. @@ -2084,51 +2099,54 @@ The following setup must occur before running each of the following test cases. #### Setup 1. Create a `MongoClient` object (referred to as `client`). + 2. Using `client`, drop the collection `keyvault.datakeys`. + 3. Using `client`, create a unique index on `keyAltNames` with a partial index filter for only documents where - `keyAltNames` exists using writeConcern "majority". + `keyAltNames` exists using writeConcern "majority". -The command should be equivalent to: + The command should be equivalent to: -```typescript -db.runCommand( - { - createIndexes: "datakeys", - indexes: [ - { - name: "keyAltNames_1", - key: { "keyAltNames": 1 }, - unique: true, - partialFilterExpression: { keyAltNames: { $exists: true } } - } - ], - writeConcern: { w: "majority" } - } -) -``` + ```typescript + db.runCommand( + { + createIndexes: "datakeys", + indexes: [ + { + name: "keyAltNames_1", + key: { "keyAltNames": 1 }, + unique: true, + partialFilterExpression: { keyAltNames: { $exists: true } } + } + ], + writeConcern: { w: "majority" } + } + ) + ``` 4. Create a `ClientEncryption` object (referred to as `client_encryption`) with `client` set as the `keyVaultClient`. + 5. Using `client_encryption`, create a data key with a `local` KMS provider and the keyAltName "def". #### Case 1: createDataKey() 1. Use `client_encryption` to create a new local data key with a keyAltName "abc" and assert the operation does not - fail. + fail. 2. Repeat Step 1 and assert the operation fails due to a duplicate key server error (error code 11000). 3. Use `client_encryption` to create a new local data key with a keyAltName "def" and assert the operation fails due to - a duplicate key server error (error code 11000). + a duplicate key server error (error code 11000). #### Case 2: addKeyAltName() 1. Use `client_encryption` to create a new local data key and assert the operation does not fail. 2. Use `client_encryption` to add a keyAltName "abc" to the key created in Step 1 and assert the operation does not - fail. + fail. 3. Repeat Step 2, assert the operation does not fail, and assert the returned key document contains the keyAltName "abc" - added in Step 2. + added in Step 2. 4. Use `client_encryption` to add a keyAltName "def" to the key created in Step 1 and assert the operation fails due to - a duplicate key server error (error code 11000). + a duplicate key server error (error code 11000). 5. Use `client_encryption` to add a keyAltName "def" to the existing key, assert the operation does not fail, and assert - the returned key document contains the keyAltName "def" added during Setup. + the returned key document contains the keyAltName "def" added during Setup. ### 14. Decryption Events @@ -2247,9 +2265,9 @@ for the field `cursor.firstBatch.encrypted`. ### 15. On-demand AWS Credentials These tests require valid AWS credentials. Refer: -[Automatic AWS Credentials](../client-side-encryption.rst#automatic-aws-credentials). +[Automatic AWS Credentials](../client-side-encryption.md#automatic-credentials). -For these cases, create a [ClientEncryption](../client-side-encryption.rst#clientencryption) object $C$ with the +For these cases, create a [ClientEncryption](../client-side-encryption.md#clientencryption) object $C$ with the following options: ```typescript @@ -2263,7 +2281,7 @@ class ClientEncryptionOpts { #### Case 1: Failure Do not run this test case in an environment where AWS credentials are available (e.g. via environment variables or a -metadata URL). (Refer: [Obtaining credentials for AWS](../../auth/auth.rst#obtaining-credentials)) +metadata URL). (Refer: [Obtaining credentials for AWS](../../auth/auth.md#obtaining-credentials)) Attempt to create a datakey with $C$ using the `"aws"` KMS provider. Expect this to fail due to a lack of KMS provider credentials. @@ -2326,55 +2344,55 @@ pairs where `srcProvider` equals `dstProvider`. 2. Create a `ClientEncryption` object named `clientEncryption1` with these options: - ```typescript - class ClientEncryptionOpts { - keyVaultClient: , - keyVaultNamespace: "keyvault.datakeys", - kmsProviders: , - } - ``` + ```typescript + class ClientEncryptionOpts { + keyVaultClient: , + keyVaultNamespace: "keyvault.datakeys", + kmsProviders: , + } + ``` 3. Call `clientEncryption1.createDataKey` with `srcProvider` and these options: - ```typescript - class DataKeyOpts { - masterKey: , - } - ``` + ```typescript + class DataKeyOpts { + masterKey: , + } + ``` - Store the return value in `keyID`. + Store the return value in `keyID`. 4. Call `clientEncryption1.encrypt` with the value "test" and these options: - ```typescript - class EncryptOpts { - keyId : keyID, - algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", - } - ``` + ```typescript + class EncryptOpts { + keyId : keyID, + algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", + } + ``` - Store the return value in `ciphertext`. + Store the return value in `ciphertext`. 5. Create a `ClientEncryption` object named `clientEncryption2` with these options: - ```typescript - class ClientEncryptionOpts { - keyVaultClient: , - keyVaultNamespace: "keyvault.datakeys", - kmsProviders: , - } - ``` + ```typescript + class ClientEncryptionOpts { + keyVaultClient: , + keyVaultNamespace: "keyvault.datakeys", + kmsProviders: , + } + ``` 6. Call `clientEncryption2.rewrapManyDataKey` with an empty `filter` and these options: - ```typescript - class RewrapManyDataKeyOpts { - provider: dstProvider, - masterKey: , - } - ``` + ```typescript + class RewrapManyDataKeyOpts { + provider: dstProvider, + masterKey: , + } + ``` - Assert that the returned `RewrapManyDataKeyResult.bulkWriteResult.modifiedCount` is 1. + Assert that the returned `RewrapManyDataKeyResult.bulkWriteResult.modifiedCount` is 1. 7. Call `clientEncryption1.decrypt` with the `ciphertext`. Assert the return value is "test". @@ -2387,30 +2405,30 @@ impossible by design to omit `RewrapManyDataKeyOpts.provider` when `RewrapManyDa 1. Create a `ClientEncryption` object named `clientEncryption` with these options: - ```typescript - class ClientEncryptionOpts { - keyVaultClient: , - keyVaultNamespace: "keyvault.datakeys", - kmsProviders: , - } - ``` + ```typescript + class ClientEncryptionOpts { + keyVaultClient: , + keyVaultNamespace: "keyvault.datakeys", + kmsProviders: , + } + ``` 2. Call `clientEncryption.rewrapManyDataKey` with an empty `filter` and these options: - ```typescript - class RewrapManyDataKeyOpts { - masterKey: {} - } - ``` + ```typescript + class RewrapManyDataKeyOpts { + masterKey: {} + } + ``` - Assert that `clientEncryption.rewrapManyDataKey` raises a client error indicating that the required - `RewrapManyDataKeyOpts.provider` field is missing. + Assert that `clientEncryption.rewrapManyDataKey` raises a client error indicating that the required + `RewrapManyDataKeyOpts.provider` field is missing. ### 17. On-demand GCP Credentials -Refer: [Automatic GCP Credentials](../client-side-encryption.rst#automatic-gcp-credentials). +Refer: [Automatic GCP Credentials](../client-side-encryption.md#obtaining-gcp-credentials). -For these cases, create a [ClientEncryption](../client-side-encryption.rst#clientencryption) object $C$ with the +For these cases, create a [ClientEncryption](../client-side-encryption.md#clientencryption) object $C$ with the following options: ```typescript @@ -2467,7 +2485,7 @@ Expect the key to be successfully created. ### 18. Azure IMDS Credentials -Refer: [Automatic Azure Credentials](../client-side-encryption.rst#obtaining-an-access-token-for-azure-key-vault) +Refer: [Automatic Azure Credentials](../client-side-encryption.md#obtaining-an-access-token-for-azure-key-vault) The test cases for IMDS communication are specially designed to not require an Azure environment, while still exercising the core of the functionality. The design of these test cases encourages an implementation to separate the concerns of @@ -2478,7 +2496,7 @@ For these IMDS credentials tests, a simple stand-in IMDS-imitating HTTP server i at `.evergreen/csfle/fake_azure.py`. `fake_azure.py` is a very simple `bottle.py` application. For the easiest use, it is recommended to execute it through `bottle.py` (which is a sibling file in the same directory): -``` +```shell python .evergreen/csfle/bottle.py fake_azure:imds ``` @@ -2486,13 +2504,13 @@ This will run the `imds` Bottle application defined in the `fake_azure` Python m command line arguments to control the bind host and TCP port (use `--help` for more information). For each test case, follow the process for obtaining the token as outlined in the -[automatic Azure credentials section](../client-side-encryption.rst#obtaining-an-access-token-for-azure-key-vault) with +[automatic Azure credentials section](../client-side-encryption.md#obtaining-an-access-token-for-azure-key-vault) with the following changes: 1. Instead of the standard IMDS TCP endpoint of `169.254.169.254:80`, communicate with the running `fake_azure` HTTP - server. + server. 2. For each test case, the behavior of the server may be controlled by attaching an additional HTTP header to the sent - request: `X-MongoDB-HTTP-TestParams`. + request: `X-MongoDB-HTTP-TestParams`. #### Case 1: Success @@ -2573,9 +2591,9 @@ a timeout. ### 19. Azure IMDS Credentials Integration Test -Refer: [Automatic Azure Credentials](../client-side-encryption.rst#obtaining-an-access-token-for-azure-key-vault) +Refer: [Automatic Azure Credentials](../client-side-encryption.md#obtaining-an-access-token-for-azure-key-vault) -For these cases, create a [ClientEncryption](../client-side-encryption.rst#clientencryption) object $C$ with the +For these cases, create a [ClientEncryption](../client-side-encryption.md#clientencryption) object $C$ with the following options: ```typescript @@ -2628,7 +2646,7 @@ Expect the key to be successfully created. ### 20. Bypass creating mongocryptd client when shared library is loaded > [!NOTE] -> IMPORTANT: If [crypt_shared](../client-side-encryption.rst#crypt_shared) is not visible to the operating system's +> IMPORTANT: If [crypt_shared](../client-side-encryption.md#crypt_shared) is not visible to the operating system's > library search mechanism, this test should be skipped. The following tests that a mongocryptd client is not created when shared library is in-use. @@ -2636,28 +2654,28 @@ The following tests that a mongocryptd client is not created when shared library 1. Start a new thread (referred to as `listenerThread`) 2. On `listenerThread`, create a TcpListener on 127.0.0.1 endpoint and port 27021. Start the listener and wait for - establishing connections. If any connection is established, then signal about this to the main thread. + establishing connections. If any connection is established, then signal about this to the main thread. - Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port - that should be free. + Drivers MAY pass a different port if they expect their testing infrastructure to be using port 27021. Pass a port + that should be free. 3. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - Configure the required options. Use the `local` KMS provider as follows: + Configure the required options. Use the `local` KMS provider as follows: - ```javascript - { "local": { "key": } } - ``` + ```javascript + { "local": { "key": } } + ``` - Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. + Configure with the `keyVaultNamespace` set to `keyvault.datakeys`. - Configure the following `extraOptions`: + Configure the following `extraOptions`: - ```javascript - { - "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000" - } - ``` + ```javascript + { + "mongocryptdURI": "mongodb://localhost:27021/?serverSelectionTimeoutMS=1000" + } + ``` 4. Use `client_encrypted` to insert the document `{"unencrypted": "test"}` into `db.coll`. @@ -2672,9 +2690,9 @@ The Automatic Data Encryption Keys tests require MongoDB server 7.0+. The tests > libmongocrypt 1.8.0 is configured to use the QEv2 protocol. For each of the following test cases, assume `DB` is a valid open database handle, and assume a -[ClientEncryption](../client-side-encryption.rst#clientencryption) object `CE` created using the following options: +[ClientEncryption](../client-side-encryption.md#clientencryption) object `CE` created using the following options: -``` +```javascript clientEncryptionOptions: { keyVaultClient: , keyVaultNamespace: "keyvault.datakeys", @@ -2703,56 +2721,56 @@ When testing `local`, set `masterKey` to `null`. #### Case 1: Simple Creation and Validation This test is the most basic to verify that -[CreateEncryptedCollection](../client-side-encryption.rst#create-encrypted-collection-helper) created a collection with +[CreateEncryptedCollection](../client-side-encryption.md#create-encrypted-collection-helper) created a collection with queryable encryption enabled. It verifies that the server rejects an attempt to insert plaintext in an encrypted fields. 1. Create a new create-collection options $Opts$ including the following: - ```typescript - { - encryptedFields: { - fields: [{ - path: "ssn", - bsonType: "string", - keyId: null - }] - } - } - ``` + ```typescript + { + encryptedFields: { + fields: [{ + path: "ssn", + bsonType: "string", + keyId: null + }] + } + } + ``` 2. Invoke $CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)$ to obtain a new collection - $Coll$. Expect success. + $Coll$. Expect success. 3. Attempt to insert the following document into `Coll`: - ```typescript - { - ssn: "123-45-6789" - } - ``` + ```typescript + { + ssn: "123-45-6789" + } + ``` 4. Expect an error from the insert operation that indicates that the document failed validation. This error indicates - that the server expects to receive an encrypted field for `ssn`, but we tried to insert a plaintext field via a - client that is unaware of the encryption requirements. + that the server expects to receive an encrypted field for `ssn`, but we tried to insert a plaintext field via a + client that is unaware of the encryption requirements. #### Case 2: Missing `encryptedFields` -The [CreateEncryptedCollection](../client-side-encryption.rst#create-encrypted-collection-helper) helper should not +The [CreateEncryptedCollection](../client-side-encryption.md#create-encrypted-collection-helper) helper should not create a regular collection if there are no `encryptedFields` for the collection being created. Instead, it should generate an error indicated that the `encryptedFields` option is missing. 1. Create a new empty create-collection options $Opts$. (i.e. it must not contain any `encryptedFields` options.) 2. Invoke $CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)$. 3. Expect the invocation to fail with an error indicating that `encryptedFields` is not defined for the collection, and - expect that no collection was created within the database. It would be *incorrect* for - [CreateEncryptedCollection](../client-side-encryption.rst#create-encrypted-collection-helper) to create a regular - collection without queryable encryption enabled. + expect that no collection was created within the database. It would be *incorrect* for + [CreateEncryptedCollection](../client-side-encryption.md#create-encrypted-collection-helper) to create a regular + collection without queryable encryption enabled. #### Case 3: Invalid `keyId` -The [CreateEncryptedCollection](../client-side-encryption.rst#create-encrypted-collection-helper) helper only inspects +The [CreateEncryptedCollection](../client-side-encryption.md#create-encrypted-collection-helper) helper only inspects `encryptedFields.fields` for `keyId` of `null`. -[CreateEncryptedCollection](../client-side-encryption.rst#create-encrypted-collection-helper) should forward all other +[CreateEncryptedCollection](../client-side-encryption.md#create-encrypted-collection-helper) should forward all other data as-is, even if it would be malformed. The server should generate an error when attempting to create a collection with such invalid settings. @@ -2762,22 +2780,22 @@ with such invalid settings. 1. Create a new create-collection options $Opts$ including the following: - ```typescript - { - encryptedFields: { - fields: [{ - path: "ssn", - bsonType: "string", - keyId: false, - }] - } - } - ``` + ```typescript + { + encryptedFields: { + fields: [{ + path: "ssn", + bsonType: "string", + keyId: false, + }] + } + } + ``` 2. Invoke $CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)$. 3. Expect an error from the server indicating a validation error at `create.encryptedFields.fields.keyId`, which must be - a UUID and not a boolean value. + a UUID and not a boolean value. #### Case 4: Insert encrypted value @@ -2785,46 +2803,39 @@ This test is continuation of the case 1 and provides a way to complete inserting 1. Create a new create-collection options $Opts$ including the following: - ```typescript - { - encryptedFields: { - fields: [{ - path: "ssn", - bsonType: "string", - keyId: null - }] - } - } - ``` + ```typescript + { + encryptedFields: { + fields: [{ + path: "ssn", + bsonType: "string", + keyId: null + }] + } + } + ``` 2. Invoke $CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)$ to obtain a new collection - $Coll$ and data key $key1$. Expect success. + $Coll$ and data key $key1$. Expect success. 3. Use $CE$ to explicitly encrypt the string "123-45-6789" using algorithm $Unindexed$ and data key $key1$. Refer result - as $encryptedPayload$. + as $encryptedPayload$. 4. Attempt to insert the following document into `Coll`: - ```typescript - { - ssn: - } - ``` + ```typescript + { + ssn: + } + ``` - Expect success. + Expect success. ### 22. Range Explicit Encryption -The Range Explicit Encryption tests require MongoDB server 8.0.0-rc14+ for SERVER-91889 and libmongocrypt containing -MONGOCRYPT-705. - -> [!NOTE] -> MongoDB Server 8.0 introduced a backwards breaking change to the Queryable Encryption (QE) range protocol: QE Range V2 -> libmongocrypt 1.11.0 is required to use the QE Range V2. - -> [!NOTE] -> MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2. -> libmongocrypt 1.8.0 is configured to use the QEv2 protocol. +The Range Explicit Encryption tests utilize Queryable Encryption (QE) range protocol V2 and require MongoDB server +8.0.0-rc14+ for [SERVER-91889](https://jira.mongodb.org/browse/SERVER-91889) and libmongocrypt 1.11.0+ for +[MONGOCRYPT-705](https://jira.mongodb.org/browse/MONGOCRYPT-705). The tests must not run against a standalone. Each of the following test cases must pass for each of the supported types (`DecimalNoPrecision`, `DecimalPrecision`, `DoublePrecision`, `DoubleNoPrecision`, `Date`, `Int`, and `Long`), unless it is stated the type should be skipped. @@ -2847,7 +2858,7 @@ as `key1Document`. Read the `"_id"` field of `key1Document` as `key1ID`. Drop and create the collection `db.explicit_encryption` using `encryptedFields` as an option. See -[FLE 2 CreateCollection() and Collection.Drop()](../client-side-encryption.md#fle-2-createcollection-and-collection-drop). +[FLE 2 CreateCollection() and Collection.Drop()](../client-side-encryption.md#create-collection-helper). Drop and create the collection `keyvault.datakeys`. @@ -2911,78 +2922,78 @@ skipped. 1. DecimalNoPrecision - ```typescript - class RangeOpts { - trimFactor: 1, - sparsity: 1, - } - ``` + ```typescript + class RangeOpts { + trimFactor: 1, + sparsity: 1, + } + ``` 2. DecimalPrecision - ```typescript - class RangeOpts { - min: { "$numberDecimal": "0" }, - max: { "$numberDecimal": "200" }, - trimFactor: 1, - sparsity: 1, - precision: 2, - } - ``` + ```typescript + class RangeOpts { + min: { "$numberDecimal": "0" }, + max: { "$numberDecimal": "200" }, + trimFactor: 1, + sparsity: 1, + precision: 2, + } + ``` 3. DoubleNoPrecision - ```typescript - class RangeOpts { - trimFactor: 1 - sparsity: 1, - } - ``` + ```typescript + class RangeOpts { + trimFactor: 1 + sparsity: 1, + } + ``` 4. DoublePrecision - ```typescript - class RangeOpts { - min: { "$numberDouble": "0" }, - max: { "$numberDouble": "200" }, - trimFactor: 1, - sparsity: 1, - precision: 2, - } - ``` + ```typescript + class RangeOpts { + min: { "$numberDouble": "0" }, + max: { "$numberDouble": "200" }, + trimFactor: 1, + sparsity: 1, + precision: 2, + } + ``` 5. Date - ```typescript - class RangeOpts { - min: {"$date": { "$numberLong": "0" } } , - max: {"$date": { "$numberLong": "200" } }, - trimFactor: 1, - sparsity: 1, - } - ``` + ```typescript + class RangeOpts { + min: {"$date": { "$numberLong": "0" } } , + max: {"$date": { "$numberLong": "200" } }, + trimFactor: 1, + sparsity: 1, + } + ``` 6. Int - ```typescript - class RangeOpts { - min: {"$numberInt": "0" } , - max: {"$numberInt": "200" }, - trimFactor: 1, - sparsity: 1, - } - ``` + ```typescript + class RangeOpts { + min: {"$numberInt": "0" } , + max: {"$numberInt": "200" }, + trimFactor: 1, + sparsity: 1, + } + ``` 7. Long - ```typescript - class RangeOpts { - min: {"$numberLong": "0" } , - max: {"$numberLong": "200" }, - trimFactor: 1, - sparsity: 1, - } - ``` + ```typescript + class RangeOpts { + min: {"$numberLong": "0" } , + max: {"$numberLong": "200" }, + trimFactor: 1, + sparsity: 1, + } + ``` #### Case 1: can decrypt a payload @@ -3296,3 +3307,109 @@ Assert the returned payload size is greater than the size of `payload_defaults`. > [!NOTE] > Do not compare the payload contents. The payloads include random data. The `trimFactor` and `sparsity` directly affect > the payload size. + +### 24. KMS Retry Tests + +The following tests that certain AWS, Azure, and GCP KMS operations are retried on transient errors. + +This test uses a mock server with configurable failpoints to simulate network failures. To start the server: + +```shell +python -u kms_failpoint_server.py --port 9003 +``` + +See the [TLS tests](#10-kms-tls-tests) for running the mock server on Evergreen. See +[the mock server implementation](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/4ba50d373652b6fb39239745664637e33e2b01e6/.evergreen/csfle/kms_failpoint_server.py) +and the +[C driver tests](https://github.com/mongodb/mongo-c-driver/blob/d934cd5de55af65220816e4fd01ce3f9c0ef1cd4/src/libmongoc/tests/test-mongoc-client-side-encryption.c#L6295) +for how to configure failpoints. + +#### Setup + +1. Start a `mongod` process with **server version 4.2.0 or later**. +2. Start the failpoint KMS server with: `python -u kms_failpoint_server.py --port 9003`. +3. Create a `MongoClient` for key vault operations. +4. Create a `ClientEncryption` object (referred to as `client_encryption`) with `keyVaultNamespace` set to + `keyvault.datakeys`. + +The failpoint server is configured using HTTP requests. Example request to simulate a network failure: + +`curl -X POST https://localhost:9003/set_failpoint/network -d '{"count": 1}' --cacert drivers-evergreen-tools/.evergreen/x509gen/ca.pem` + +To simulate an HTTP failure, replace `network` with `http`. + +When the following test cases request setting `masterKey`, use the following values based on the KMS provider: + +For "aws": + +```javascript +{ + "region": "foo", + "key": "bar", + "endpoint": "127.0.0.1:9003", +} +``` + +For "azure": + +```javascript +{ + "keyVaultEndpoint": "127.0.0.1:9003", + "keyName": "foo", +} +``` + +For "gcp": + +```javascript +{ + "projectId": "foo", + "location": "bar", + "keyRing": "baz", + "keyName": "qux", + "endpoint": "127.0.0.1:9003" +} +``` + +#### Case 1: createDataKey and encrypt with TCP retry + +1. Configure the mock server to simulate one network failure. +2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to succeed. Store the returned key + ID in a variable named `keyId`. +3. Configure the mock server to simulate another network failure. +4. Call `clientEncryption.encrypt` with the following `EncryptOpts` to encrypt the int32 value `123` with the newly + created key: + ```typescript + class EncryptOpts { + keyId : , + algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", + } + ``` + Expect this to succeed. + +Repeat this test with the `azure` and `gcp` masterKeys. + +#### Case 2: createDataKey and encrypt with HTTP retry + +1. Configure the mock server to simulate one HTTP failure. +2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to succeed. Store the returned key + ID in a variable named `keyId`. +3. Configure the mock server to simulate another HTTP failure. +4. Call `clientEncryption.encrypt` with the following `EncryptOpts` to encrypt the int32 value `123` with the newly + created key: + ```typescript + class EncryptOpts { + keyId : , + algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic", + } + ``` + Expect this to succeed. + +Repeat this test with the `azure` and `gcp` masterKeys. + +#### Case 3: createDataKey fails after too many retries + +1. Configure the mock server to simulate four network failures. +2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to fail. + +Repeat this test with the `azure` and `gcp` masterKeys. diff --git a/src/test/spec/json/client-side-encryption/benchmarks.md b/src/test/spec/json/client-side-encryption/benchmarks.md index 453d60846..64e1694a8 100644 --- a/src/test/spec/json/client-side-encryption/benchmarks.md +++ b/src/test/spec/json/client-side-encryption/benchmarks.md @@ -10,7 +10,7 @@ due to the cost of calling between the native language and the C library. A handle to libmongocrypt (`mongocrypt_t`) is needed for the benchmark. In the public driver API, `mongocrypt_t` is an implementation detail contained in a `MongoClient`. The bindings API may more directly interface `mongocrypt_t`. Example: the Java bindings API contains a -[MongoCrypt class](https://github.com/mongodb/libmongocrypt/blob/master/bindings/java/mongocrypt/src/main/java/com/mongodb/crypt/capi/MongoCrypt.java) +[MongoCrypt class](https://github.com/mongodb/mongo-java-driver/blob/main/mongodb-crypt/src/main/com/mongodb/internal/crypt/capi/MongoCrypt.java) closely wrapping the `mongocrypt_t`. If possible, drivers are encouraged to use the bindings API and mock responses from the MongoDB server. This may help to @@ -25,11 +25,11 @@ Set up the benchmark data: - Create a data key with the "local" KMS provider. - Encrypt 1500 string values of the form `value 0001`, `value 0002`, `value 0003`, ... with the algorithm - `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`. + `AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic`. - Create a document of the form: - `{ "key0001": , "key0002": , "key0003": }`. + `{ "key0001": , "key0002": , "key0003": }`. - Create a handle to `mongocrypt_t`. This may be through the bindings API (preferred) or through a `MongoClient` - configured with `AutoEncryptionOpts`. + configured with `AutoEncryptionOpts`. Warm up the benchmark: diff --git a/src/test/spec/json/client-side-encryption/legacy/fle2v2-Rangev2-Compact.yml b/src/test/spec/json/client-side-encryption/legacy/fle2v2-Rangev2-Compact.yml index 9ec6c3e58..4a36409d3 100644 --- a/src/test/spec/json/client-side-encryption/legacy/fle2v2-Rangev2-Compact.yml +++ b/src/test/spec/json/client-side-encryption/legacy/fle2v2-Rangev2-Compact.yml @@ -90,4 +90,4 @@ tests: } } encryptionInformation: *encryptionInformation - command_name: compactStructuredEncryptionData + command_name: compactStructuredEncryptionData \ No newline at end of file diff --git a/src/test/spec/json/client-side-encryption/legacy/keyCache.json b/src/test/spec/json/client-side-encryption/legacy/keyCache.json new file mode 100644 index 000000000..9dfaa680f --- /dev/null +++ b/src/test/spec/json/client-side-encryption/legacy/keyCache.json @@ -0,0 +1,270 @@ +{ + "runOn": [ + { + "minServerVersion": "4.1.10" + } + ], + "database_name": "default", + "collection_name": "default", + "data": [], + "json_schema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + }, + "key_vault_data": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ], + "tests": [ + { + "description": "Insert with deterministic encryption, then find it", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + }, + "keyExpirationMS": 1 + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + }, + { + "name": "wait", + "object": "testRunner", + "arguments": { + "ms": 50 + } + }, + { + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "result": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "expectations": [ + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "command_name": "listCollections" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + }, + "command_name": "insert" + } + }, + { + "command_started_event": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "command_name": "find" + } + } + ], + "outcome": { + "collection": { + "data": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/src/test/spec/json/client-side-encryption/legacy/keyCache.yml b/src/test/spec/json/client-side-encryption/legacy/keyCache.yml new file mode 100644 index 000000000..3af117ca8 --- /dev/null +++ b/src/test/spec/json/client-side-encryption/legacy/keyCache.yml @@ -0,0 +1,69 @@ +runOn: + - minServerVersion: "4.1.10" +database_name: &database_name "default" +collection_name: &collection_name "default" + +data: [] +json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} +key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}] + +tests: + - description: "Insert with deterministic encryption, then find it" + clientOptions: + autoEncryptOpts: + kmsProviders: + aws: {} # Credentials filled in from environment. + keyExpirationMS: 1 + operations: + - name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + - name: wait + object: testRunner + arguments: + ms: 50 # Wait long enough to account for coarse time resolution on Windows (CDRIVER-4526). + - name: find + arguments: + filter: { _id: 1 } + result: [*doc0] + expectations: + # Auto encryption will request the collection info. + - command_started_event: + command: + listCollections: 1 + filter: + name: *collection_name + command_name: listCollections + # Then key is fetched from the key vault. + - command_started_event: + command: + find: datakeys + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + $db: keyvault + readConcern: { level: "majority" } + command_name: find + - command_started_event: + command: + insert: *collection_name + documents: + - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } + ordered: true + command_name: insert + - command_started_event: + command: + find: *collection_name + filter: { _id: 1 } + command_name: find + # The cache has expired and the key must be fetched again + - command_started_event: + command: + find: datakeys + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + $db: keyvault + readConcern: { level: "majority" } + command_name: find + outcome: + collection: + # Outcome is checked using a separate MongoClient without auto encryption. + data: + - *doc0_encrypted \ No newline at end of file diff --git a/src/test/spec/json/client-side-encryption/unified/keyCache.json b/src/test/spec/json/client-side-encryption/unified/keyCache.json new file mode 100644 index 000000000..a39701e28 --- /dev/null +++ b/src/test/spec/json/client-side-encryption/unified/keyCache.json @@ -0,0 +1,198 @@ +{ + "description": "keyCache-explicit", + "schemaVersion": "1.22", + "runOnRequirements": [ + { + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "clientEncryption": { + "id": "clientEncryption0", + "clientEncryptionOpts": { + "keyVaultClient": "client0", + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "OCTP9uKPPmvuqpHlqq83gPk4U6rUPxKVRRyVtrjFmVjdoa4Xzm1SzUbr7aIhNI42czkUBmrCtZKF31eaaJnxEBkqf0RFukA9Mo3NEHQWgAQ2cn9duOcRbaFUQo2z0/rB" + } + }, + "keyExpirationMS": 1 + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + }, + "keyAltNames": [], + "keyMaterial": { + "$binary": { + "base64": "iocBkhO3YBokiJ+FtxDTS71/qKXQ7tSWhWbcnFTXBcMjarsepvALeJ5li+SdUd9ePuatjidxAdMo7vh1V2ZESLMkQWdpPJ9PaJjA67gKQKbbbB4Ik5F2uKjULvrMBnFNVRMup4JNUwWFQJpqbfMveXnUVcD06+pUpAkml/f+DSXrV3e5rxciiNVtz03dAG8wJrsKsFXWj6vTjFhsfknyBA==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + } + ], + "tests": [ + { + "description": "decrypt, wait, and decrypt again", + "operations": [ + { + "name": "decrypt", + "object": "clientEncryption0", + "arguments": { + "value": { + "$binary": { + "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", + "subType": "06" + } + } + }, + "expectResult": "foobar" + }, + { + "name": "wait", + "object": "testRunner", + "arguments": { + "ms": 50 + } + }, + { + "name": "decrypt", + "object": "clientEncryption0", + "arguments": { + "value": { + "$binary": { + "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", + "subType": "06" + } + } + }, + "expectResult": "foobar" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ] + } + ] +} diff --git a/src/test/spec/json/client-side-encryption/unified/keyCache.yml b/src/test/spec/json/client-side-encryption/unified/keyCache.yml new file mode 100644 index 000000000..d6e747ba0 --- /dev/null +++ b/src/test/spec/json/client-side-encryption/unified/keyCache.yml @@ -0,0 +1,85 @@ +description: keyCache-explicit + +schemaVersion: "1.22" + +runOnRequirements: + - csfle: true + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - clientEncryption: + id: &clientEncryption0 clientEncryption0 + clientEncryptionOpts: + keyVaultClient: *client0 + keyVaultNamespace: keyvault.datakeys + kmsProviders: + "local" : { key: "OCTP9uKPPmvuqpHlqq83gPk4U6rUPxKVRRyVtrjFmVjdoa4Xzm1SzUbr7aIhNI42czkUBmrCtZKF31eaaJnxEBkqf0RFukA9Mo3NEHQWgAQ2cn9duOcRbaFUQo2z0/rB" } + keyExpirationMS: 1 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name keyvault + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name datakeys + +initialData: + - databaseName: *database0Name + collectionName: *collection0Name + documents: + - { + "_id": { + "$binary": { + "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "subType": "04" + } + }, + "keyAltNames": [], + "keyMaterial": { + "$binary": { + "base64": "iocBkhO3YBokiJ+FtxDTS71/qKXQ7tSWhWbcnFTXBcMjarsepvALeJ5li+SdUd9ePuatjidxAdMo7vh1V2ZESLMkQWdpPJ9PaJjA67gKQKbbbB4Ik5F2uKjULvrMBnFNVRMup4JNUwWFQJpqbfMveXnUVcD06+pUpAkml/f+DSXrV3e5rxciiNVtz03dAG8wJrsKsFXWj6vTjFhsfknyBA==", + "subType": "00" + } + }, + "creationDate": {"$date": {"$numberLong": "1552949630483"}}, + "updateDate": {"$date": {"$numberLong": "1552949630483"}}, + "status": {"$numberInt": "0"}, + "masterKey": {"provider": "local"} + } + +tests: + - description: decrypt, wait, and decrypt again + operations: + - name: decrypt + object: *clientEncryption0 + arguments: + value: { "$binary" : { "base64" : "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", "subType" : "06" } } + expectResult: "foobar" + - name: wait + object: testRunner + arguments: + ms: 50 # Wait long enough to account for coarse time resolution on Windows (CDRIVER-4526). + - name: decrypt + object: *clientEncryption0 + arguments: + value: { "$binary" : { "base64" : "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", "subType" : "06" } } + expectResult: "foobar" + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: datakeys + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'a+YWzdygTAG62/cNUkqZiQ==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + $db: keyvault + readConcern: { level: "majority" } + - commandStartedEvent: + command: + find: datakeys + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'a+YWzdygTAG62/cNUkqZiQ==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + $db: keyvault + readConcern: { level: "majority" } diff --git a/src/test/spec/unified_runner/test_file.rs b/src/test/spec/unified_runner/test_file.rs index 1f0bf2c3e..9450a4cec 100644 --- a/src/test/spec/unified_runner/test_file.rs +++ b/src/test/spec/unified_runner/test_file.rs @@ -345,6 +345,12 @@ pub(crate) struct ClientEncryptionOpts { pub(crate) key_vault_client: String, pub(crate) key_vault_namespace: crate::Namespace, pub(crate) kms_providers: HashMap, + #[serde( + default, + rename = "keyExpirationMS", + deserialize_with = "serde_util::deserialize_duration_option_from_u64_millis" + )] + pub(crate) key_cache_expiration: Option, } /// Messages used for communicating with test runner "threads". diff --git a/src/test/spec/unified_runner/test_runner.rs b/src/test/spec/unified_runner/test_runner.rs index 48120e5bd..b966fdf33 100644 --- a/src/test/spec/unified_runner/test_runner.rs +++ b/src/test/spec/unified_runner/test_runner.rs @@ -67,7 +67,7 @@ const SKIPPED_OPERATIONS: &[&str] = &[ ]; static MIN_SPEC_VERSION: Version = Version::new(1, 0, 0); -static MAX_SPEC_VERSION: Version = Version::new(1, 21, 0); +static MAX_SPEC_VERSION: Version = Version::new(1, 22, 0); pub(crate) type EntityMap = HashMap; @@ -616,13 +616,15 @@ impl TestRunner { .unwrap() .clone(); let kms_providers = fill_kms_placeholders(opts.kms_providers.clone()); - let client_enc = crate::client_encryption::ClientEncryption::new( + let client_encryption = crate::client_encryption::ClientEncryption::builder( kv_client, opts.key_vault_namespace.clone(), kms_providers, ) + .key_cache_expiration(opts.key_cache_expiration) + .build() .unwrap(); - (id, Entity::ClientEncryption(Arc::new(client_enc))) + (id, Entity::ClientEncryption(Arc::new(client_encryption))) } }; self.insert_entity(&id, entity).await; diff --git a/src/test/spec/v2_runner.rs b/src/test/spec/v2_runner.rs index 0298f179a..b6834f241 100644 --- a/src/test/spec/v2_runner.rs +++ b/src/test/spec/v2_runner.rs @@ -462,6 +462,7 @@ impl<'a> OpRunner<'a> { .unwrap(); self.fail_point_guards.push(guard); } + "wait" => operation.execute().await, other => panic!("unknown operation: {}", other), } return None; diff --git a/src/test/spec/v2_runner/operation.rs b/src/test/spec/v2_runner/operation.rs index 2b5956866..8f19b3d88 100644 --- a/src/test/spec/v2_runner/operation.rs +++ b/src/test/spec/v2_runner/operation.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, convert::TryInto, fmt::Debug, ops::Deref}; +use std::{collections::HashMap, convert::TryInto, fmt::Debug, ops::Deref, time::Duration}; use futures::{future::BoxFuture, stream::TryStreamExt, FutureExt}; use serde::{de::Deserializer, Deserialize}; @@ -46,6 +46,10 @@ use crate::{ use super::{OpRunner, OpSessions}; pub(crate) trait TestOperation: Debug + Send + Sync { + fn execute(&self) -> BoxFuture<'_, ()> { + todo!() + } + fn execute_on_collection<'a>( &'a self, _collection: &'a Collection, @@ -287,6 +291,7 @@ impl<'de> Deserialize<'de> for Operation { "assertIndexNotExists" => deserialize_op::(definition.arguments), "watch" => deserialize_op::(definition.arguments), "withTransaction" => deserialize_op::(definition.arguments), + "wait" => deserialize_op::(definition.arguments), _ => Ok(Box::new(UnimplementedOperation) as Box), } .map_err(|e| serde::de::Error::custom(format!("{}", e)))?; @@ -1450,6 +1455,21 @@ impl TestOperation for WithTransaction { } } +#[derive(Debug, Deserialize)] +struct Wait { + ms: u64, +} + +impl TestOperation for Wait { + fn execute(&self) -> BoxFuture<'_, ()> { + async move { + let duration = Duration::from_millis(self.ms); + tokio::time::sleep(duration).await; + } + .boxed() + } +} + #[derive(Debug, Deserialize)] pub(super) struct UnimplementedOperation;