PYTHON-1332 - Test lsid with all commands

Author: ajdavis

doc/changelog.rst

@@ -20,6 +20,13 @@ Highlights include:
 
 Breaking changes include:
 
+- Certain commands, such as ``ismaster`` and ``ping``, that could be used
+ to check whether a server is available without requiring authentication, now
+ raise :exc:`~pymongo.errors.OperationFailure` if executed without
+ authenticating on a MongoDB 3.6+ server started with ``--auth``. (This is
+ because all commands are now sent with a session id, whether a
+ :class:`~pymongo.client_session.ClientSession` is used or not, and all
+ commands with a session id require auth.)
 - BSON binary subtype 4 is decoded using RFC-4122 byte order regardless
  of the UUID representation. This is a change in behavior for applications
  that use UUID representation :data:`bson.binary.JAVA_LEGACY` or

test/__init__.py

@@ -18,6 +18,7 @@
 import os
 import socket
 import sys
+import time
 import warnings
 
 try:
@@ -88,14 +89,14 @@ def is_server_resolvable():
 
 
 def _connect(host, port, **kwargs):
- try:
-  client = pymongo.MongoClient(
-  host, port, serverSelectionTimeoutMS=100, **kwargs)
- client.admin.command('ismaster') # Can we connect?
- # If connected, then return client with default timeout
- return pymongo.MongoClient(host, port, **kwargs)
- except pymongo.errors.ConnectionFailure:
-  return None
+ client = pymongo.MongoClient(host, port, **kwargs)
+ start = time.time()
+ while not client.nodes:
+ time.sleep(0.05)
+ if time.time() - start > 0.1:
+  return None
+
+ return client
 
 
 class client_knobs(object):
@@ -147,11 +148,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
 
 def _all_users(db):
- if Version.from_client(db.client).at_least(2, 5, 3, -1):
- return set(u['user']
- for u in db.command('usersInfo').get('users', []))
- else:
- return set(u['user'] for u in db.system.users.find())
+ return set(u['user'] for u in db.command('usersInfo').get('users', []))
 
 
 class ClientContext(object):
@@ -190,32 +187,6 @@ def __init__(self):
 
  if self.client:
  self.connected = True
- ismaster = self.client.admin.command('ismaster')
- self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
-
- if 'setName' in ismaster:
- self.replica_set_name = ismaster['setName']
- self.is_rs = True
- # It doesn't matter which member we use as the seed here.
- self.client = pymongo.MongoClient(
- host,
- port,
- replicaSet=self.replica_set_name,
- **self.ssl_client_options)
- # Get the authoritative ismaster result from the primary.
- self.ismaster = self.client.admin.command('ismaster')
- nodes = [partition_node(node.lower())
- for node in self.ismaster.get('hosts', [])]
- nodes.extend([partition_node(node.lower())
- for node in self.ismaster.get('passives', [])])
- nodes.extend([partition_node(node.lower())
- for node in self.ismaster.get('arbiters', [])])
- self.nodes = set(nodes)
- else:
- self.ismaster = ismaster
- self.nodes = set([(host, port)])
- self.w = len(self.ismaster.get("hosts", [])) or 1
- self.version = Version.from_client(self.client)
 
  try:
  self.cmd_line = self.client.admin.command('getCmdLineOpts')
@@ -232,10 +203,7 @@ def __init__(self):
  if self.auth_enabled:
  # See if db_user already exists.
  if not self._check_user_provided():
- roles = {}
- if self.version.at_least(2, 5, 3, -1):
- roles = {'roles': ['root']}
- self.client.admin.add_user(db_user, db_pwd, **roles)
+ self.client.admin.add_user(db_user, db_pwd, roles=['root'])
 
  self.client = _connect(host,
  port,
@@ -247,6 +215,43 @@ def __init__(self):
  # May not have this if OperationFailure was raised earlier.
  self.cmd_line = self.client.admin.command('getCmdLineOpts')
 
+ self.ismaster = ismaster = self.client.admin.command('isMaster')
+ self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
+
+ if 'setName' in ismaster:
+ self.replica_set_name = ismaster['setName']
+ self.is_rs = True
+ if self.auth_enabled:
+ # It doesn't matter which member we use as the seed here.
+ self.client = pymongo.MongoClient(
+ host,
+ port,
+ username=db_user,
+ password=db_pwd,
+ replicaSet=self.replica_set_name,
+ **self.ssl_client_options)
+ else:
+ self.client = pymongo.MongoClient(
+ host,
+ port,
+ replicaSet=self.replica_set_name,
+ **self.ssl_client_options)
+
+ # Get the authoritative ismaster result from the primary.
+ self.ismaster = self.client.admin.command('ismaster')
+ nodes = [partition_node(node.lower())
+ for node in self.ismaster.get('hosts', [])]
+ nodes.extend([partition_node(node.lower())
+ for node in self.ismaster.get('passives', [])])
+ nodes.extend([partition_node(node.lower())
+ for node in self.ismaster.get('arbiters', [])])
+ self.nodes = set(nodes)
+ else:
+ self.ismaster = ismaster
+ self.nodes = set([(host, port)])
+ self.w = len(self.ismaster.get("hosts", [])) or 1
+ self.version = Version.from_client(self.client)
+
  if 'enableTestCommands=1' in self.cmd_line['argv']:
  self.test_commands_enabled = True
  elif 'parsed' in self.cmd_line:
@@ -487,6 +492,7 @@ def require_sessions(self, func):
  "Sessions not supported",
  func=func)
 
+
 # Reusable client context
 client_context = ClientContext()
 
@@ -499,6 +505,10 @@ class IntegrationTest(unittest.TestCase):
  def setUpClass(cls):
  cls.client = client_context.client
  cls.db = cls.client.pymongo_test
+ if client_context.auth_enabled:
+ cls.credentials = {'username': db_user, 'password': db_pwd}
+ else:
+ cls.credentials = {}
 
 
 # Use assertRaisesRegex if available, otherwise use Python 2.7's

test/test_change_stream.py

@@ -33,6 +33,7 @@
 from pymongo.command_cursor import CommandCursor
 from pymongo.errors import (InvalidOperation, OperationFailure,
  ServerSelectionTimeoutError)
+from pymongo.message import _CursorAddress
 from pymongo.read_concern import ReadConcern
 
 from test import client_context, unittest, IntegrationTest
@@ -189,7 +190,8 @@ def test_resume_on_error(self):
  self.insert_and_check(change_stream, {'_id': 1})
  # Cause a cursor not found error on the next getMore.
  cursor = change_stream._cursor
- self.client._close_cursor_now(cursor.cursor_id, cursor.address)
+ address = _CursorAddress(cursor.address, self.coll.full_name)
+ self.client._close_cursor_now(cursor.cursor_id, address)
  self.insert_and_check(change_stream, {'_id': 2})
 
  def test_does_not_resume_on_server_error(self):
@@ -235,7 +237,8 @@ def raise_error():
  self.insert_and_check(change_stream, {'_id': 1})
  # Cause a cursor not found error on the next getMore.
  cursor = change_stream._cursor
- self.client._close_cursor_now(cursor.cursor_id, cursor.address)
+ address = _CursorAddress(cursor.address, self.coll.full_name)
+ self.client._close_cursor_now(cursor.cursor_id, address)
  cursor.close = raise_error
  self.insert_and_check(change_stream, {'_id': 2})
 

test/test_client.py

@@ -354,23 +354,27 @@ def test_constants(self):
  port are not overloaded.
  """
  host, port = client_context.host, client_context.port
+ kwargs = client_context.ssl_client_options.copy()
+ if client_context.auth_enabled:
+ kwargs['username'] = db_user
+ kwargs['password'] = db_pwd
+
  # Set bad defaults.
  MongoClient.HOST = "somedomainthatdoesntexist.org"
  MongoClient.PORT = 123456789
  with self.assertRaises(AutoReconnect):
  connected(MongoClient(serverSelectionTimeoutMS=10,
- **client_context.ssl_client_options))
+ **kwargs))
 
  # Override the defaults. No error.
- connected(MongoClient(host, port,
- **client_context.ssl_client_options))
+ connected(MongoClient(host, port, **kwargs))
 
  # Set good defaults.
  MongoClient.HOST = host
  MongoClient.PORT = port
 
  # No error.
- connected(MongoClient(**client_context.ssl_client_options))
+ connected(MongoClient(**kwargs))
 
  def test_init_disconnected(self):
  host, port = client_context.host, client_context.port
@@ -1051,7 +1055,8 @@ def test_stale_getmore(self):
  serverSelectionTimeoutMS=100)
  client._send_message_with_response(
  operation=message._GetMore('pymongo_test', 'collection',
- 101, 1234, client.codec_options),
+ 101, 1234, client.codec_options,
+ None),
  address=('not-a-member', 27017))
 
  def test_heartbeat_frequency_ms(self):

test/test_collection.py

@@ -2252,8 +2252,8 @@ def test_find_regex(self):
  self.assertTrue(isinstance(doc['r'], Regex))
 
  def test_find_command_generation(self):
- cmd = _gen_find_command(
- 'coll', {'$query': {'foo': 1}, '$dumb': 2}, None, 0, 0, 0, None)
+ cmd = _gen_find_command('coll', {'$query': {'foo': 1}, '$dumb': 2},
+  None, 0, 0, 0, None, None)
  self.assertEqual(
  cmd.to_dict(),
  SON([('find', 'coll'),

test/test_common.py

@@ -25,7 +25,7 @@
 from bson.objectid import ObjectId
 from pymongo.errors import OperationFailure
 from pymongo.write_concern import WriteConcern
-from test import client_context, unittest, IntegrationTest
+from test import client_context, IntegrationTest
 from test.utils import connected, rs_or_single_client, single_client
 
 
@@ -195,7 +195,8 @@ def test_mongo_client(self):
 
  # Equality tests
  direct = connected(single_client(w=0))
- direct2 = connected(single_client("mongodb://%s/?w=0" % (pair,)))
+ direct2 = connected(single_client("mongodb://%s/?w=0" % (pair,),
+ **self.credentials))
  self.assertEqual(direct, direct2)
  self.assertFalse(direct != direct2)
 

test/test_cursor.py

@@ -27,7 +27,6 @@
 from bson.py3compat import PY3
 from bson.son import SON
 from pymongo import (monitoring,
- MongoClient,
  ASCENDING,
  DESCENDING,
  ALL,
@@ -52,12 +51,7 @@
  long = int
 
 
-class TestCursorNoConnect(unittest.TestCase):
-
- @classmethod
- def setUpClass(cls):
- cls.db = MongoClient(connect=False).test
-
+class TestCursor(IntegrationTest):
  def test_deepcopy_cursor_littered_with_regexes(self):
  cursor = self.db.test.find({
  "x": re.compile("^hmmm.*"),
@@ -134,9 +128,6 @@ def test_add_remove_option(self):
  cursor.remove_option(128)
  self.assertEqual(0, cursor._Cursor__query_flags)
 
-
-class TestCursor(IntegrationTest):
-
  def test_add_remove_option_exhaust(self):
  # Exhaust - which mongos doesn't support
  if client_context.is_mongos:

test/test_cursor_manager.py

@@ -19,7 +19,6 @@
 
 sys.path[0:0] = [""]
 
-from pymongo.command_cursor import CommandCursor
 from pymongo.cursor_manager import CursorManager
 from pymongo.errors import CursorNotFound
 from pymongo.message import _CursorAddress
@@ -97,37 +96,6 @@ def raises_cursor_not_found():
  wait_until(raises_cursor_not_found, 'close cursor')
  self.assertTrue(self.close_was_called)
 
- def test_cursor_transfer(self):
-
- # This is just a test, don't try this at home...
-
- client = rs_or_single_client()
- db = client.pymongo_test
-
- db.test.delete_many({})
- db.test.insert_many([{'_id': i} for i in range(200)])
-
- class CManager(CursorManager):
- def __init__(self, client):
- super(CManager, self).__init__(client)
-
- def close(self, dummy, dummy2):
- # Do absolutely nothing...
- pass
-
- client.set_cursor_manager(CManager)
- docs = []
- cursor = db.test.find().batch_size(10)
- docs.append(next(cursor))
- cursor.close()
- docs.extend(cursor)
- self.assertEqual(len(docs), 10)
- cmd_cursor = {'id': cursor.cursor_id, 'firstBatch': []}
- ccursor = CommandCursor(cursor.collection, cmd_cursor,
- cursor.address, retrieved=cursor.retrieved)
- docs.extend(ccursor)
- self.assertEqual(len(docs), 200)
-
 
 if __name__ == "__main__":
  unittest.main()

test/test_grid_file.py

@@ -232,7 +232,12 @@ def test_grid_out_cursor_options(self):
 
  cursor = GridOutCursor(self.db.fs, {})
  cursor_clone = cursor.clone()
- self.assertEqual(cursor_clone.__dict__, cursor.__dict__)
+ 
+ cursor_dict = cursor.__dict__.copy()
+ cursor_dict.pop('_Cursor__session')
+ cursor_clone_dict = cursor_clone.__dict__.copy()
+ cursor_clone_dict.pop('_Cursor__session')
+ self.assertEqual(cursor_dict, cursor_clone_dict)
 
  self.assertRaises(NotImplementedError, cursor.add_option, 0)
  self.assertRaises(NotImplementedError, cursor.remove_option, 0)

test/test_replica_set_client.py

@@ -14,7 +14,6 @@
 
 """Test the mongo_replica_set_client module."""
 
-import socket
 import sys
 import warnings
 import time
@@ -161,32 +160,6 @@ def test_properties(self):
  self.assertEqual(c.max_bson_size, 16777216)
  c.close()
 
- @client_context.require_secondaries_count(1)
- def test_auto_reconnect_exception_when_read_preference_is_secondary(self):
- c = MongoClient(
- client_context.pair,
- replicaSet=self.name,
- serverSelectionTimeoutMS=100)
- db = c.pymongo_test
-
- def raise_socket_error(*args, **kwargs):
- raise socket.error
-
- # In Jython socket.socket is a function, not a class.
- sock = socket.socket()
- klass = sock.__class__
- old_sendall = klass.sendall
- klass.sendall = raise_socket_error
-
- try:
- cursor = db.get_collection(
- "test", read_preference=ReadPreference.SECONDARY).find()
- self.assertRaises(AutoReconnect, cursor.next)
- finally:
- klass.sendall = old_sendall
- # Silence resource warnings.
- sock.close()
-
  @client_context.require_secondaries_count(1)
  def test_timeout_does_not_mark_member_down(self):
  # If a query times out, the client shouldn't mark the member "down".