CSHARP-3162: Testing Data Lake with Drivers

Author: MikalaiMazurenka

build.cake

@@ -237,6 +237,24 @@ Task("TestAtlasConnectivity")
  );
 });
 
+Task("TestAtlasDataLake")
+ .IsDependentOn("Build")
+ .DoesForEach(
+ GetFiles("./**/MongoDB.Driver.Tests.csproj"),
+ testProject =>
+ {
+ DotNetCoreTest(
+ testProject.FullPath,
+ new DotNetCoreTestSettings {
+ NoBuild = true,
+ NoRestore = true,
+ Configuration = configuration,
+ ArgumentCustomization = args => args.Append("-- RunConfiguration.TargetPlatform=x64"),
+ Filter = "Category=\"AtlasDataLake\""
+ }
+ );
+ });
+
 Task("TestOcsp")
  .IsDependentOn("Build")
  .DoesForEach(

evergreen/evergreen.yml

@@ -229,6 +229,19 @@ functions:
  params:
  file: mo-expansion.yml
 
+ bootstrap-mongohoused:
+ - command: shell.exec
+ params:
+ script: |
+ DRIVERS_TOOLS="${DRIVERS_TOOLS}" \
+ sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
+ - command: shell.exec
+ params:
+ background: true
+ script: |
+ DRIVERS_TOOLS="${DRIVERS_TOOLS}" \
+ sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
+
  run-tests:
  - command: shell.exec
  type: test
@@ -421,6 +434,15 @@ functions:
  ${PREPARE_SHELL}
  evergreen/run-mongodb-aws-test.sh
 
+ run-atlas-data-lake-test:
+ - command: shell.exec
+ type: test
+ params:
+ working_dir: mongo-csharp-driver
+ script: |
+ ${PREPARE_SHELL}
+ evergreen/run-atlas-data-lake-test.sh
+
  run-ocsp-test:
  - command: shell.exec
  type: test
@@ -657,6 +679,11 @@ tasks:
  - func: run-aws-auth-test-with-aws-EC2-credentials
  # ECS test is skipped untill testing on Linux becomes possible
 
+ - name: atlas-data-lake-test
+ commands:
+ - func: bootstrap-mongohoused
+ - func: run-atlas-data-lake-test
+
  - name: publish-snapshot
  depends_on:
  - variant: ".tests-variant"
@@ -1172,6 +1199,13 @@ buildvariants:
  tasks:
  - name: atlas-connectivity-tests
 
+- name: atlas-data-lake-test
+ display_name: "Atlas Data Lake Tests"
+ run_on:
+ - windows-64-vs2017-test
+ tasks:
+ - name: atlas-data-lake-test
+
 # - name: plain-auth-tests
 # display_name: "PLAIN (LDAP) Auth tests"
 # run_on:

evergreen/run-atlas-data-lake-test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit # Exit the script with error if any of the commands fail
+
+############################################
+# Main Program #
+############################################
+
+echo "Running Atlas Data Lake driver tests"
+
+export MONGODB_URI="mongodb://mhuser:pencil@localhost"
+export ATLAS_DATA_LAKE_TESTS_ENABLED=true
+
+powershell.exe .\\build.ps1 -target TestAtlasDataLake

tests/MongoDB.Driver.Tests/JsonDrivenTests/JsonDrivenFindTest.cs

@@ -101,6 +101,10 @@ protected override void SetArgument(string name, BsonValue value)
  _options.Limit = value.ToInt32();
  return;
 
+ case "projection":
+ _options.Projection = (ProjectionDefinition<BsonDocument, BsonDocument>)value;
+ return;
+
  case "result":
  ParseExpectedResult(value.IsBsonArray ? value.AsBsonArray : new BsonArray(new[] { value }));
  return;

tests/MongoDB.Driver.Tests/Specifications/Runner/MongoClientJsonDrivenTestRunnerBase.cs

@@ -246,6 +246,16 @@ protected virtual void ExecuteOperations(IMongoClient client, Dictionary<string,
  }
  }
 
+ protected virtual string GetCollectionName(BsonDocument definition)
+ {
+ return definition[CollectionNameKey].AsString;
+ }
+
+ protected virtual string GetDatabaseName(BsonDocument definition)
+ {
+ return definition[DatabaseNameKey].AsString;
+ }
+
  protected virtual void InsertData(IMongoClient client, string databaseName, string collectionName, BsonDocument shared)
  {
  if (shared.Contains(DataKey))
@@ -469,8 +479,8 @@ private void SetupAndRunTest(BsonDocument shared, BsonDocument test)
 
  CustomDataValidation(shared, test);
 
- DatabaseName = shared[DatabaseNameKey].AsString;
- CollectionName = shared[CollectionNameKey].AsString;
+ DatabaseName = GetDatabaseName(shared);
+ CollectionName = GetCollectionName(shared);
 
  var client = CreateClientForTestSetup();
  TestInitialize(client, test, shared);

tests/MongoDB.Driver.Tests/Specifications/atlas-data-lake/AtlasDataLakeTestRunner.cs

@@ -0,0 +1,95 @@
+/* Copyright 2020-present MongoDB Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.TestHelpers.JsonDrivenTests;
+using MongoDB.Bson.TestHelpers.XunitExtensions;
+using MongoDB.Driver.Tests.Specifications.Runner;
+using Xunit;
+
+namespace MongoDB.Driver.Tests.Specifications.atlas_data_lake
+{
+ [Trait("Category", "AtlasDataLake")]
+ public class AtlasDataLakeTestRunner : MongoClientJsonDrivenTestRunnerBase
+ {
+ protected override string[] ExpectedSharedColumns => new[] { "_path", "database_name", "collection_name", "tests" };
+ protected override string[] ExpectedTestColumns => new[] { "description", "operations", "expectations", "async" };
+
+ [SkippableTheory]
+ [ClassData(typeof(TestCaseFactory))]
+ public void Run(JsonDrivenTestCase testCase)
+ {
+ RequireEnvironment.Check().EnvironmentVariable("ATLAS_DATA_LAKE_TESTS_ENABLED");
+
+ SetupAndRunTest(testCase);
+ }
+
+ protected override void CreateCollection(IMongoClient client, string databaseName, string collectionName, BsonDocument test, BsonDocument shared)
+ {
+ // do nothing
+ }
+
+ protected override void DropCollection(MongoClient client, string databaseName, string collectionName, BsonDocument test, BsonDocument shared)
+ {
+ // do nothing
+ }
+
+ protected override string GetCollectionName(BsonDocument definition)
+ {
+ if (definition.TryGetValue(CollectionNameKey, out var collectionName))
+ {
+ return collectionName.AsString;
+ }
+ else
+ {
+ return DriverTestConfiguration.CollectionNamespace.CollectionName;
+ }
+ }
+
+ protected override string GetDatabaseName(BsonDocument definition)
+ {
+ if (definition.TryGetValue(DatabaseNameKey, out var databaseName))
+ {
+ return databaseName.AsString;
+ }
+ else
+ {
+ return DriverTestConfiguration.DatabaseNamespace.DatabaseName;
+ }
+ }
+
+ // nested types
+ private class TestCaseFactory : JsonDrivenTestCaseFactory
+ {
+ // protected properties
+ protected override string PathPrefix => "MongoDB.Driver.Tests.Specifications.atlas_data_lake.tests.";
+
+ // protected methods
+ protected override IEnumerable<JsonDrivenTestCase> CreateTestCases(BsonDocument document)
+ {
+ foreach (var testCase in base.CreateTestCases(document))
+ {
+ foreach (var async in new[] { false, true })
+ {
+ var name = $"{testCase.Name}:async={async}";
+ var test = testCase.Test.DeepClone().AsBsonDocument.Add("async", async);
+ yield return new JsonDrivenTestCase(name, testCase.Shared, test);
+ }
+ }
+ }
+ }
+ }
+}

tests/MongoDB.Driver.Tests/Specifications/atlas-data-lake/prose-tests/AtlasDataLakeProseTests.cs

@@ -0,0 +1,121 @@
+/* Copyright 2020-present MongoDB Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System.Linq;
+using FluentAssertions;
+using MongoDB.Bson;
+using MongoDB.Bson.TestHelpers.XunitExtensions;
+using MongoDB.Driver.Core;
+using MongoDB.Driver.Core.Events;
+using MongoDB.Driver.Core.Misc;
+using MongoDB.Driver.Core.TestHelpers.XunitExtensions;
+using Xunit;
+
+namespace MongoDB.Driver.Tests.Specifications.atlas_data_lake.prose_tests
+{
+ [Trait("Category", "AtlasDataLake")]
+ public class AtlasDataLakeProseTests
+ {
+ [SkippableFact]
+ public void Driver_should_connect_to_AtlasDataLake_without_authentication()
+ {
+ RequireEnvironment.Check().EnvironmentVariable("ATLAS_DATA_LAKE_TESTS_ENABLED");
+
+ using (var client = DriverTestConfiguration.CreateDisposableClient())
+ {
+ client.GetDatabase("admin").RunCommand<BsonDocument>(new BsonDocument("ping", 1));
+ }
+ }
+
+ [SkippableFact]
+ public void Driver_should_connect_to_AtlasDataLake_with_SCRAM_SHA_1()
+ {
+ RequireEnvironment.Check().EnvironmentVariable("ATLAS_DATA_LAKE_TESTS_ENABLED");
+ RequireServer.Check().Supports(Feature.ScramSha1Authentication);
+
+ var connectionString = CoreTestConfiguration.ConnectionString;
+ var username = connectionString.Username;
+ var password = connectionString.Password;
+ var source = connectionString.AuthSource;
+
+ var settings = DriverTestConfiguration.Client.Settings.Clone();
+ settings.Credential = MongoCredential.FromComponents(mechanism: "SCRAM-SHA-1", source, username, password);
+
+ using (var client = DriverTestConfiguration.CreateDisposableClient(settings))
+ {
+ client.GetDatabase("admin").RunCommand<BsonDocument>(new BsonDocument("ping", 1));
+ }
+ }
+
+ [SkippableFact]
+ public void Driver_should_connect_to_AtlasDataLake_with_SCRAM_SHA_256()
+ {
+ RequireEnvironment.Check().EnvironmentVariable("ATLAS_DATA_LAKE_TESTS_ENABLED");
+ RequireServer.Check().Supports(Feature.ScramSha256Authentication);
+
+ var connectionString = CoreTestConfiguration.ConnectionString;
+ var username = connectionString.Username;
+ var password = connectionString.Password;
+ var source = connectionString.AuthSource;
+
+ var settings = DriverTestConfiguration.Client.Settings.Clone();
+ settings.Credential = MongoCredential.FromComponents(mechanism: "SCRAM-SHA-256", source, username, password);
+
+ using (var client = DriverTestConfiguration.CreateDisposableClient(settings))
+ {
+ client.GetDatabase("admin").RunCommand<BsonDocument>(new BsonDocument("ping", 1));
+ }
+ }
+
+ [SkippableFact]
+ public void KillCursors_should_return_expected_result()
+ {
+ RequireEnvironment.Check().EnvironmentVariable("ATLAS_DATA_LAKE_TESTS_ENABLED");
+ RequireServer.Check().Supports(Feature.KillCursorsCommand);
+
+ var databaseName = "test";
+ var collectionName = "driverdata";
+
+ var eventCapturer = new EventCapturer()
+ .Capture<CommandStartedEvent>(x => "killCursors" == x.CommandName)
+ .Capture<CommandSucceededEvent>(x => new[] { "killCursors", "find" }.Contains(x.CommandName));
+
+ using (var client = DriverTestConfiguration.CreateDisposableClient(eventCapturer))
+ {
+ var cursor = client
+ .GetDatabase(databaseName)
+ .GetCollection<BsonDocument>(collectionName)
+ .Find(new BsonDocument(), new FindOptions { BatchSize = 2 })
+ .ToCursor();
+
+ var findCommandSucceededEvent = eventCapturer.Events.OfType<CommandSucceededEvent>().First(x => x.CommandName == "find");
+ var findCommandResult = findCommandSucceededEvent.Reply;
+ var cursorId = findCommandResult["cursor"]["id"].AsInt64;
+ var cursorNamespace = CollectionNamespace.FromFullName(findCommandResult["cursor"]["ns"].AsString);
+
+ cursor.Dispose();
+
+ var killCursorsCommandStartedEvent = eventCapturer.Events.OfType<CommandStartedEvent>().First(x => x.CommandName == "killCursors");
+ var killCursorsCommandSucceededEvent = eventCapturer.Events.OfType<CommandSucceededEvent>().First(x => x.CommandName == "killCursors");
+ var killCursorsStartedCommand = killCursorsCommandStartedEvent.Command;
+
+ cursorNamespace.DatabaseNamespace.DatabaseName.Should().Be(killCursorsCommandStartedEvent.DatabaseNamespace.DatabaseName);
+ cursorNamespace.CollectionName.Should().Be(killCursorsStartedCommand["killCursors"].AsString);
+ cursorId.Should().Be(killCursorsStartedCommand["cursors"][0].AsInt64);
+ cursorId.Should().Be(killCursorsCommandSucceededEvent.Reply["cursorsKilled"][0].AsInt64);
+ }
+ }
+ }
+}