CSHARP-4106: Add ClientEncryption entity and Key Management API operations to Unified Test Format. (mongodb#826)

Author: DmitryLukyanov

build.cake

@@ -420,7 +420,7 @@ Task("TestCsfleWithMockedKmsNet472").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard20").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard21").IsDependentOn("TestCsfleWithMockedKms");
 
-Task("TestMongocryptd")
+Task("TestCsfleWithMongocryptd")
  .IsDependentOn("Build")
  .DoesForEach(
  items: GetFiles("./**/*.Tests.csproj"),
@@ -443,9 +443,9 @@ Task("TestMongocryptd")
  );
  });
 
-Task("TestMongocryptdNet472").IsDependentOn("TestMongocryptd");
-Task("TestMongocryptdNetStandard20").IsDependentOn("TestMongocryptd");
-Task("TestMongocryptdNetStandard21").IsDependentOn("TestMongocryptd");
+Task("TestCsfleWithMongocryptdNet472").IsDependentOn("TestCsfleWithMongocryptd");
+Task("TestCsfleWithMongocryptdNetStandard20").IsDependentOn("TestCsfleWithMongocryptd");
+Task("TestCsfleWithMongocryptdNetStandard21").IsDependentOn("TestCsfleWithMongocryptd");
 
 Task("Docs")
  .IsDependentOn("ApiDocs")

evergreen/evergreen.yml

@@ -47,9 +47,12 @@ functions:
 
  export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
 
- # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
  if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+ # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
  export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+ else
+ # non windows OSs don't have dotnet in the PATH
+ export PATH=$PATH:/usr/share/dotnet
  fi
 
  export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
@@ -80,6 +83,21 @@ functions:
  # See what we've done
  cat expansion.yml
 
+ # Add CSFLE variables that shouldn't be output to the logs
+ cat <<EOT >> expansion.yml
+ PREPARE_CSFLE: |
+ set +o xtrace # Disable tracing.
+ export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
+ export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
+ export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
+ export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
+ export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
+ export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
+ export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY} 
+ set -o xtrace # Enable tracing.
+ EOT
+ # Do not output expansion.yml contents after this point
+
  # Load the expansion file to make an evergreen variable with the current unique version
  - command: expansions.update
  params:
@@ -282,13 +300,7 @@ functions:
  working_dir: mongo-csharp-driver
  script: |
  set +x
- export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
- export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
- export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
- export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
- export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
- export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
- export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+ ${PREPARE_CSFLE}
  . ./evergreen/set-virtualenv.sh
  . ./evergreen/set-temp-fle-aws-creds.sh
  ${PREPARE_SHELL}
@@ -310,23 +322,19 @@ functions:
  OS=${OS} \
  evergreen/cleanup-test-resources.sh
 
- run-csfle-tests-with-mocked-kms:
+ run-csfle-with-mocked-kms-tests:
  - command: shell.exec
  type: test
  params:
  working_dir: "mongo-csharp-driver"
  script: |
  set +x
- export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
- export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
- export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
- export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
- export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
- export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
- export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+ ${PREPARE_CSFLE}
  export KMS_MOCK_SERVERS_ENABLED=true
  ${PREPARE_SHELL}
  set +o xtrace
+ OS=${OS} \
+ . ./evergreen/fetch-crypt_shared-library.sh 
  OS=${OS} \
  evergreen/add-ca-certs.sh
  AUTH=${AUTH} \
@@ -341,20 +349,14 @@ functions:
  OS=${OS} \
  evergreen/cleanup-test-resources.sh
 
- run-mongocryptd-tests:
+ run-csfle-with-mongocryptd-tests:
  - command: shell.exec
  type: test
  params:
  working_dir: mongo-csharp-driver
  script: |
  set +x
- export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
- export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
- export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
- export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
- export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
- export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
- export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+ ${PREPARE_CSFLE}
  . ./evergreen/set-virtualenv.sh
  . ./evergreen/set-temp-fle-aws-creds.sh
  ${PREPARE_SHELL}
@@ -368,8 +370,7 @@ functions:
  COMPRESSOR=${COMPRESSOR} \
  CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \
  REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
- TEST_MONGOCRYPTD="true" \
- TARGET="TestMongocryptd" \
+ TARGET="TestCsfleWithMongocryptd" \
  FRAMEWORK=${FRAMEWORK} \
  evergreen/run-tests.sh
  echo "Skipping certificate removal..."
@@ -872,51 +873,51 @@ tasks:
  vars:
  FRAMEWORK: netstandard21
 
- - name: test-mongocryptd-net472
+ - name: test-csfle-with-mongocryptd-net472
  commands:
  - func: bootstrap-mongo-orchestration
- - func: run-mongocryptd-tests
+ - func: run-csfle-with-mongocryptd-tests
  vars:
  FRAMEWORK: net472
 
- - name: test-mongocryptd-netstandard20
+ - name: test-csfle-with-mongocryptd-netstandard20
  commands:
  - func: bootstrap-mongo-orchestration
- - func: run-mongocryptd-tests
+ - func: run-csfle-with-mongocryptd-tests
  vars:
  FRAMEWORK: netstandard20
 
- - name: test-mongocryptd-netstandard21
+ - name: test-csfle-with-mongocryptd-netstandard21
  commands:
  - func: bootstrap-mongo-orchestration
- - func: run-mongocryptd-tests
+ - func: run-csfle-with-mongocryptd-tests
  vars:
  FRAMEWORK: netstandard21
 
- - name: test-kms-tls-mocked-net472
+ - name: test-csfle-with-mocked-kms-tls-net472
  commands:
  - func: start-kms-mock-servers
  - func: start-kms-kmip-server
  - func: bootstrap-mongo-orchestration
- - func: run-csfle-tests-with-mocked-kms
+ - func: run-csfle-with-mocked-kms-tests
  vars:
  FRAMEWORK: net472
 
- - name: test-kms-tls-mocked-netstandard20
+ - name: test-csfle-with-mocked-kms-tls-netstandard20
  commands:
  - func: start-kms-mock-servers
  - func: start-kms-kmip-server
  - func: bootstrap-mongo-orchestration
- - func: run-csfle-tests-with-mocked-kms
+ - func: run-csfle-with-mocked-kms-tests
  vars:
  FRAMEWORK: netstandard20
 
- - name: test-kms-tls-mocked-netstandard21
+ - name: test-csfle-with-mocked-kms-tls-netstandard21
  commands:
  - func: start-kms-mock-servers
  - func: start-kms-kmip-server
  - func: bootstrap-mongo-orchestration
- - func: run-csfle-tests-with-mocked-kms
+ - func: run-csfle-with-mocked-kms-tests
  vars:
  FRAMEWORK: netstandard21
 
@@ -1721,40 +1722,40 @@ buildvariants:
  matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
  display_name: "CSFLE Mocked KMS ${version} ${os}"
  tasks:
- - name: test-kms-tls-mocked-net472
- - name: test-kms-tls-mocked-netstandard20
- - name: test-kms-tls-mocked-netstandard21
+ - name: test-csfle-with-mocked-kms-tls-net472
+ - name: test-csfle-with-mocked-kms-tls-netstandard20
+ - name: test-csfle-with-mocked-kms-tls-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-linux"
  matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
  display_name: "CSFLE Mocked KMS ${version} ${os}"
  tasks:
- - name: test-kms-tls-mocked-netstandard20
- - name: test-kms-tls-mocked-netstandard21
+ - name: test-csfle-with-mocked-kms-tls-netstandard20
+ - name: test-csfle-with-mocked-kms-tls-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-macOS"
  matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
  display_name: "CSFLE Mocked KMS ${version} ${os}"
  tasks:
- - name: test-kms-tls-mocked-netstandard21
+ - name: test-csfle-with-mocked-kms-tls-netstandard21
 
-- matrix_name: "csfle1-windows"
+- matrix_name: "csfle-with-mongocryptd-windows"
  matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
- display_name: "mongocryptd ${version} ${os}"
+ display_name: "CSFLE with mongocryptd ${version} ${os}"
  tasks:
- - name: test-mongocryptd-net472
- - name: test-mongocryptd-netstandard20
- - name: test-mongocryptd-netstandard21
+ - name: test-csfle-with-mongocryptd-net472
+ - name: test-csfle-with-mongocryptd-netstandard20
+ - name: test-csfle-with-mongocryptd-netstandard21
 
-- matrix_name: "csfle1-linux"
+- matrix_name: "csfle-with-mongocryptd-linux"
  matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
- display_name: "mongocryptd ${version} ${os}"
+ display_name: "CSFLE with mongocryptd ${version} ${os}"
  tasks:
- - name: test-mongocryptd-netstandard20
- - name: test-mongocryptd-netstandard21
+ - name: test-csfle-with-mongocryptd-netstandard20
+ - name: test-csfle-with-mongocryptd-netstandard21
 
-- matrix_name: "csfle1-macOS"
+- matrix_name: "csfle-with-mongocryptd-macOS"
  matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
- display_name: "mongocryptd ${version} ${os}"
+ display_name: "CSFLE with mongocryptd ${version} ${os}"
  tasks:
- - name: test-mongocryptd-netstandard21
+ - name: test-csfle-with-mongocryptd-netstandard21

evergreen/fetch-crypt_shared-library.sh

@@ -14,7 +14,7 @@ set -o errexit # Exit the script with an error if any of the commands fail
 
 
 PYTHON=$(OS=${OS} ${PROJECT_DIRECTORY}/evergreen/get-python-path.sh)
-$PYTHON -u ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --out ${DRIVERS_TOOLS}/evergreen/csfle --version 6.0.0-rc8
+$PYTHON -u ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --out ${DRIVERS_TOOLS}/evergreen/csfle --version 6.0.0-rc13
 
 if [[ "$OS" =~ Windows|windows ]]; then
  export CRYPT_SHARED_LIB_PATH="${DRIVERS_TOOLS}/evergreen/csfle/bin/mongo_crypt_v1.dll"

evergreen/run-tests.sh

@@ -69,7 +69,19 @@ provision_compressor () {
 # Main Program #
 ############################################
 echo "CRYPT_SHARED_LIB_PATH:" $CRYPT_SHARED_LIB_PATH
-echo "TEST_MONGOCRYPTD:" $TEST_MONGOCRYPTD
+
+if [ "$TARGET" == "TestCsfleWithMongocryptd" ]; then
+ if [ ! -z "${CRYPT_SHARED_LIB_PATH}" ]; then
+ echo "CRYPT_SHARED_LIB_PATH must be unassigned for CSFLE tests with mongocryptd, but was ${CRYPT_SHARED_LIB_PATH}" 1>&2 # write to stderr
+ exit 1 
+ fi
+else
+ if [ -z "${CRYPT_SHARED_LIB_PATH}" ]; then
+ echo "CRYPT_SHARED_LIB_PATH must be assigned, but wasn't" 1>&2 # write to stderr"
+ exit 1 
+ fi
+fi
+
 echo "Initial MongoDB URI:" $MONGODB_URI
 echo "Framework: " $FRAMEWORK
 

src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj

@@ -177,7 +177,7 @@
  <ItemGroup>
  <PackageReference Include="DnsClient" Version="1.6.1" />
  <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.6.2" PrivateAssets="All" />
- <PackageReference Include="MongoDB.Libmongocrypt" Version="1.5.3" />
+ <PackageReference Include="MongoDB.Libmongocrypt" Version="1.5.4" />
  <PackageReference Include="SharpCompress" Version="0.30.1" />
  <PackageReference Include="System.Buffers" Version="4.5.1" />
  <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">

src/MongoDB.Driver.Core/ServerErrorCode.cs

@@ -28,6 +28,7 @@ internal enum ServerErrorCode
  FailedToSatisfyReadPreference = 133,
  HostNotFound = 7,
  HostUnreachable = 6,
+ DuplicateKey = 11000,
  Interrupted = 11601,
  InterruptedAtShutdown = 11600,
  InterruptedDueToReplStateChange = 11602,

src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs

@@ -56,7 +56,7 @@ IMongoClient CreateInternalClient()
  // private fields
  private readonly IMongoClient _internalClient;
  private readonly IMongoClient _metadataClient;
- private readonly IMongoClient _mongocryptdClient;
+ private readonly Lazy<IMongoClient> _mongocryptdClient;
  private readonly MongocryptdFactory _mongocryptdFactory;
 
  // constructors
@@ -75,15 +75,20 @@ private AutoEncryptionLibMongoCryptController(
  _internalClient = internalClient; // can be null
  _metadataClient = metadataClient; // can be null
  _mongocryptdFactory = new MongocryptdFactory(autoEncryptionOptions.ExtraOptions, autoEncryptionOptions.BypassQueryAnalysis);
- _mongocryptdClient = _mongocryptdFactory.CreateMongocryptdClient();
+ _mongocryptdClient = new Lazy<IMongoClient>(() => _mongocryptdFactory.CreateMongocryptdClient(), isThreadSafe: true);
  }
 
  // internal properties
  /// <summary>
- /// this property is used by DisposableMongoClient.Dispose to unregister the internal cluster.
+ /// This property is used by DisposableMongoClient.Dispose to unregister the internal cluster.
  /// </summary>
  internal IMongoClient InternalClient => _internalClient;
 
+ /// <summary>
+ /// This property is used by DisposableMongoClient.Dispose to unregister the mongocryptd cluster.
+ /// </summary>
+ internal IMongoClient MongoCryptdClient => _mongocryptdClient.IsValueCreated ? _mongocryptdClient.Value : null;
+
  // public methods
  public byte[] DecryptFields(byte[] encryptedDocumentBytes, CancellationToken cancellationToken)
  {
@@ -217,7 +222,7 @@ private async Task ProcessNeedCollectionInfoStateAsync(CryptContext context, str
 
  private void ProcessNeedMongoMarkingsState(CryptContext context, string databaseName, CancellationToken cancellationToken)
  {
- var database = _mongocryptdClient.GetDatabase(databaseName);
+ var database = _mongocryptdClient.Value.GetDatabase(databaseName);
  var commandBytes = context.GetOperation().ToArray();
  var commandDocument = new RawBsonDocument(commandBytes);
  var command = new BsonDocumentCommand<BsonDocument>(commandDocument);
@@ -241,7 +246,7 @@ private void ProcessNeedMongoMarkingsState(CryptContext context, string database
 
  private async Task ProcessNeedMongoMarkingsStateAsync(CryptContext context, string databaseName, CancellationToken cancellationToken)
  {
- var database = _mongocryptdClient.GetDatabase(databaseName);
+ var database = _mongocryptdClient.Value.GetDatabase(databaseName);
  var commandBytes = context.GetOperation().ToArray();
  var commandDocument = new RawBsonDocument(commandBytes);
  var command = new BsonDocumentCommand<BsonDocument>(commandDocument);
@@ -268,7 +273,7 @@ private void WaitForMongocryptdReady()
  var stopwatch = Stopwatch.StartNew();
  while (stopwatch.Elapsed < TimeSpan.FromSeconds(5))
  {
- var clusterDescription = _mongocryptdClient.Cluster?.Description;
+ var clusterDescription = _mongocryptdClient.Value.Cluster?.Description;
  var mongocryptdServer = clusterDescription?.Servers?.FirstOrDefault();
  if (mongocryptdServer != null && mongocryptdServer.Type != ServerType.Unknown)
  {
@@ -283,7 +288,7 @@ private async Task WaitForMongocryptdReadyAsync()
  var stopwatch = Stopwatch.StartNew();
  while (stopwatch.Elapsed < TimeSpan.FromSeconds(5))
  {
- var clusterDescription = _mongocryptdClient.Cluster?.Description;
+ var clusterDescription = _mongocryptdClient.Value.Cluster?.Description;
  var mongocryptdServer = clusterDescription?.Servers?.FirstOrDefault();
  if (mongocryptdServer != null && mongocryptdServer.Type != ServerType.Unknown)
  {

src/MongoDB.Driver/Encryption/AutoEncryptionOptions.cs

@@ -271,14 +271,14 @@ public override string ToString()
 
  // internal methods
  internal CryptClientSettings ToCryptClientSettings() =>
-  new CryptClientSettings(
-  _bypassQueryAnalysis,
-  ExtraOptions.GetValueOrDefault<string, string, object>("cryptSharedLibPath"),
-  cryptSharedLibSearchPath: _bypassAutoEncryption ? null : "$SYSTEM",
-  _encryptedFieldsMap,
-  ExtraOptions.GetValueOrDefault<bool?, string, object>("cryptSharedLibRequired"),
-  _kmsProviders,
-  _schemaMap);
+ new CryptClientSettings(
+ _bypassQueryAnalysis,
+ ExtraOptions.GetValueOrDefault<string, string, object>("cryptSharedLibPath"),
+ cryptSharedLibSearchPath: _bypassAutoEncryption ? null : "$SYSTEM",
+ _encryptedFieldsMap,
+ ExtraOptions.GetValueOrDefault<bool?, string, object>("cryptSharedLibRequired"),
+ _kmsProviders,
+ _schemaMap);
 
  // private methods
  private bool ExtraOptionsEquals(IReadOnlyDictionary<string, object> x, IReadOnlyDictionary<string, object> y)