PYTHON-1307 Remove SONManipulator APIs (mongodb#557)

Author: ShaneHarvey

doc/api/pymongo/collection.rst

@@ -47,8 +47,8 @@
  .. automethod:: aggregate
  .. automethod:: aggregate_raw_batches
  .. automethod:: watch
- .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None)
- .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+ .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
+ .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)
  .. automethod:: find_one(filter=None, *args, **kwargs)
  .. automethod:: find_one_and_delete
  .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, hint=None, session=None, **kwargs)

doc/api/pymongo/cursor.rst

@@ -15,7 +15,7 @@
  .. autoattribute:: EXHAUST
  :annotation:
 
- .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+ .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
  :members:
 
  .. describe:: c[index]
@@ -24,4 +24,4 @@
 
  .. automethod:: __getitem__
 
- .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+ .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)

doc/api/pymongo/index.rst

@@ -47,7 +47,6 @@ Sub-modules:
  read_concern
  read_preferences
  results
- son_manipulator
  server_api
  uri_parser
  write_concern

doc/api/pymongo/son_manipulator.rst

@@ -30,11 +30,11 @@ Breaking Changes in 4.0
 - Removed :meth:`pymongo.collection.Collection.parallel_scan`.
 - Removed :meth:`pymongo.collection.Collection.ensure_index`.
 - Removed :meth:`pymongo.collection.Collection.reindex`.
-- Removed :meth:`pymongo.collection.Collection.save`
-- Removed :meth:`pymongo.collection.Collection.insert`
-- Removed :meth:`pymongo.collection.Collection.update`
-- Removed :meth:`pymongo.collection.Collection.remove`
-- Removed :meth:`pymongo.collection.Collection.find_and_modify`
+- Removed :meth:`pymongo.collection.Collection.save`.
+- Removed :meth:`pymongo.collection.Collection.insert`.
+- Removed :meth:`pymongo.collection.Collection.update`.
+- Removed :meth:`pymongo.collection.Collection.remove`.
+- Removed :meth:`pymongo.collection.Collection.find_and_modify`.
 - Removed :meth:`pymongo.mongo_client.MongoClient.close_cursor`. Use
  :meth:`pymongo.cursor.Cursor.close` instead.
 - Removed :meth:`pymongo.mongo_client.MongoClient.kill_cursors`.
@@ -43,6 +43,21 @@ Breaking Changes in 4.0
 - Removed :meth:`pymongo.mongo_client.MongoClient.set_cursor_manager`.
 - Removed :mod:`pymongo.thread_util`.
 - Removed :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`.
+- Removed :mod:`pymongo.son_manipulator`,
+ :class:`pymongo.son_manipulator.SONManipulator`,
+ :class:`pymongo.son_manipulator.ObjectIdInjector`,
+ :class:`pymongo.son_manipulator.ObjectIdShuffler`,
+ :class:`pymongo.son_manipulator.AutoReference`,
+ :class:`pymongo.son_manipulator.NamespaceInjector`,
+ :meth:`pymongo.database.Database.add_son_manipulator`,
+ :attr:`pymongo.database.Database.outgoing_copying_manipulators`,
+ :attr:`pymongo.database.Database.outgoing_manipulators`,
+ :attr:`pymongo.database.Database.incoming_copying_manipulators`, and
+ :attr:`pymongo.database.Database.incoming_manipulators`.
+- Removed the ``manipulate`` parameter from
+ :meth:`~pymongo.collection.Collection.find`,
+ :meth:`~pymongo.collection.Collection.find_one`, and
+ :meth:`~pymongo.cursor.Cursor`.
 
 Notable improvements
 ....................

doc/changelog.rst

@@ -274,6 +274,38 @@ can be changed to this::
 
 .. _reIndex command: https://docs.mongodb.com/manual/reference/command/reIndex/
 
+SONManipulator is removed
+-------------------------
+
+Removed :mod:`pymongo.son_manipulator`,
+:class:`pymongo.son_manipulator.SONManipulator`,
+:class:`pymongo.son_manipulator.ObjectIdInjector`,
+:class:`pymongo.son_manipulator.ObjectIdShuffler`,
+:class:`pymongo.son_manipulator.AutoReference`,
+:class:`pymongo.son_manipulator.NamespaceInjector`,
+:meth:`pymongo.database.Database.add_son_manipulator`,
+:attr:`pymongo.database.Database.outgoing_copying_manipulators`,
+:attr:`pymongo.database.Database.outgoing_manipulators`,
+:attr:`pymongo.database.Database.incoming_copying_manipulators`, and
+:attr:`pymongo.database.Database.incoming_manipulators`.
+
+Removed the ``manipulate`` parameter from
+:meth:`~pymongo.collection.Collection.find`,
+:meth:`~pymongo.collection.Collection.find_one`, and
+:meth:`~pymongo.cursor.Cursor`.
+
+The :class:`pymongo.son_manipulator.SONManipulator` API has limitations as a
+technique for transforming your data and was deprecated in PyMongo 3.0.
+Instead, it is more flexible and straightforward to transform outgoing
+documents in your own code before passing them to PyMongo, and transform
+incoming documents after receiving them from PyMongo.
+
+Alternatively, if your application uses the ``SONManipulator`` API to convert
+custom types to BSON, the :class:`~bson.codec_options.TypeCodec` and
+:class:`~bson.codec_options.TypeRegistry` APIs may be a suitable alternative.
+For more information, see the
+:doc:`custom type example <examples/custom_type>`.
+
 Removed features with no migration path
 ---------------------------------------
 

doc/migrate-to-pymongo4.rst

@@ -478,9 +478,9 @@ def execute_no_results(self, sock_info, generator):
  sock_info,
  operation['q'],
  doc,
- operation['upsert'],
- check_keys,
- operation['multi'],
+ upsert=operation['upsert'],
+ check_keys=check_keys,
+ multi=operation['multi'],
  write_concern=write_concern,
  op_id=op_id,
  ordered=self.ordered,

pymongo/bulk.py

@@ -554,15 +554,9 @@ def _legacy_write(self, sock_info, name, cmd, op_id,
 
  def _insert_one(
  self, doc, ordered,
- check_keys, manipulate, write_concern, op_id, bypass_doc_val,
+ check_keys, write_concern, op_id, bypass_doc_val,
  session):
  """Internal helper for inserting a single document."""
- if manipulate:
- doc = self.__database._apply_incoming_manipulators(doc, self)
- if not isinstance(doc, RawBSONDocument) and '_id' not in doc:
- doc['_id'] = ObjectId()
- doc = self.__database._apply_incoming_copying_manipulators(doc,
- self)
  write_concern = write_concern or self.write_concern
  acknowledged = write_concern.acknowledged
  command = SON([('insert', self.name),
@@ -646,7 +640,7 @@ def insert_one(self, document, bypass_document_validation=False,
  write_concern = self._write_concern_for(session)
  return InsertOneResult(
  self._insert_one(
- document, ordered=True, check_keys=True, manipulate=False,
+ document, ordered=True, check_keys=True,
  write_concern=write_concern, op_id=None,
  bypass_doc_val=bypass_document_validation, session=session),
  write_concern.acknowledged)
@@ -712,14 +706,12 @@ def gen():
  return InsertManyResult(inserted_ids, write_concern.acknowledged)
 
  def _update(self, sock_info, criteria, document, upsert=False,
- check_keys=True, multi=False, manipulate=False,
+ check_keys=True, multi=False,
  write_concern=None, op_id=None, ordered=True,
  bypass_doc_val=False, collation=None, array_filters=None,
  hint=None, session=None, retryable_write=False):
  """Internal update / replace helper."""
  common.validate_boolean("upsert", upsert)
- if manipulate:
- document = self.__database._fix_incoming(document, self)
  collation = validate_collation_or_none(collation)
  write_concern = write_concern or self.write_concern
  acknowledged = write_concern.acknowledged
@@ -801,15 +793,15 @@ def _update(self, sock_info, criteria, document, upsert=False,
 
  def _update_retryable(
  self, criteria, document, upsert=False,
- check_keys=True, multi=False, manipulate=False,
+ check_keys=True, multi=False,
  write_concern=None, op_id=None, ordered=True,
  bypass_doc_val=False, collation=None, array_filters=None,
  hint=None, session=None):
  """Internal update / replace helper."""
  def _update(session, sock_info, retryable_write):
  return self._update(
  sock_info, criteria, document, upsert=upsert,
- check_keys=check_keys, multi=multi, manipulate=manipulate,
+ check_keys=check_keys, multi=multi,
  write_concern=write_concern, op_id=op_id, ordered=ordered,
  bypass_doc_val=bypass_doc_val, collation=collation,
  array_filters=array_filters, hint=hint, session=session,
@@ -1346,8 +1338,6 @@ def find(self, *args, **kwargs):
  oplogReplay query flag. Default: False.
  - `batch_size` (optional): Limits the number of documents returned in
  a single batch.
- - `manipulate` (optional): **DEPRECATED** - If True, apply any
- outgoing SON manipulators before returning. Default: True.
  - `collation` (optional): An instance of
  :class:`~pymongo.collation.Collation`. This option is only supported
  on MongoDB 3.4 and above.

pymongo/collection.py

@@ -268,8 +268,7 @@ def _try_next(self, get_more_allowed):
  if not len(self.__data) and not self.__killed and get_more_allowed:
  self._refresh()
  if len(self.__data):
- coll = self.__collection
- return coll.database._fix_outgoing(self.__data.popleft(), coll)
+ return self.__data.popleft()
  else:
  return None
 

pymongo/command_cursor.py

@@ -108,7 +108,7 @@ def __init__(self, collection, filter=None, projection=None, skip=0,
  limit=0, no_cursor_timeout=False,
  cursor_type=CursorType.NON_TAILABLE,
  sort=None, allow_partial_results=False, oplog_replay=False,
- modifiers=None, batch_size=0, manipulate=True,
+ modifiers=None, batch_size=0,
  collation=None, hint=None, max_scan=None, max_time_ms=None,
  max=None, min=None, return_key=False, show_record_id=False,
  snapshot=False, comment=None, session=None,
@@ -181,7 +181,6 @@ def __init__(self, collection, filter=None, projection=None, skip=0,
  self.__max_await_time_ms = None
  self.__max = max
  self.__min = min
- self.__manipulate = manipulate
  self.__collation = validate_collation_or_none(collation)
  self.__return_key = return_key
  self.__show_record_id = show_record_id
@@ -281,7 +280,7 @@ def _clone(self, deepcopy=True, base=None):
  values_to_clone = ("spec", "projection", "skip", "limit",
  "max_time_ms", "max_await_time_ms", "comment",
  "max", "min", "ordering", "explain", "hint",
- "batch_size", "max_scan", "manipulate",
+ "batch_size", "max_scan",
  "query_flags", "modifiers", "collation", "empty",
  "show_record_id", "return_key", "allow_disk_use",
  "snapshot", "exhaust")
@@ -1198,12 +1197,7 @@ def next(self):
  if self.__empty:
  raise StopIteration
  if len(self.__data) or self._refresh():
- if self.__manipulate:
- _db = self.__collection.database
- return _db._fix_outgoing(self.__data.popleft(),
- self.__collection)
- else:
- return self.__data.popleft()
+ return self.__data.popleft()
  else:
  raise StopIteration
 
@@ -1277,15 +1271,8 @@ def __init__(self, *args, **kwargs):
 
  .. mongodoc:: cursors
  """
- manipulate = kwargs.get('manipulate')
- kwargs['manipulate'] = False
  super(RawBatchCursor, self).__init__(*args, **kwargs)
 
- # Throw only after cursor's initialized, to prevent errors in __del__.
- if manipulate:
- raise InvalidOperation(
- "Cannot use RawBatchCursor with manipulate=True")
-
  def _unpack_response(self, response, cursor_id, codec_options,
  user_fields=None, legacy_response=False):
  return response.raw_response(cursor_id)