PYTHON-1044 - Fix up unknown BSON type handing

Author: behackett

bson/__init__.py

@@ -93,7 +93,14 @@
 _UNPACK_TIMESTAMP = struct.Struct("<II").unpack
 
 
-def _get_int(data, position, dummy0, dummy1):
+def _raise_unknown_type(element_type, element_name):
+ """Unknown type helper."""
+ raise InvalidBSON("Detected unknown BSON type %r for fieldname '%s'. Are "
+ "you using the latest driver version?" % (
+ element_type, element_name))
+
+
+def _get_int(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON int32 to python int."""
  end = position + 4
  return _UNPACK_INT(data[position:end])[0], end
@@ -106,13 +113,13 @@ def _get_c_string(data, position, opts):
  opts.unicode_decode_error_handler, True)[0], end + 1
 
 
-def _get_float(data, position, dummy0, dummy1):
+def _get_float(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON double to python float."""
  end = position + 8
  return _UNPACK_FLOAT(data[position:end])[0], end
 
 
-def _get_string(data, position, obj_end, opts):
+def _get_string(data, position, obj_end, opts, dummy):
  """Decode a BSON string to python unicode string."""
  length = _UNPACK_INT(data[position:position + 4])[0]
  position += 4
@@ -125,7 +132,7 @@ def _get_string(data, position, obj_end, opts):
  opts.unicode_decode_error_handler, True)[0], end + 1
 
 
-def _get_object(data, position, obj_end, opts):
+def _get_object(data, position, obj_end, opts, dummy):
  """Decode a BSON subdocument to opts.document_class or bson.dbref.DBRef."""
  obj_size = _UNPACK_INT(data[position:position + 4])[0]
  end = position + obj_size - 1
@@ -146,7 +153,7 @@ def _get_object(data, position, obj_end, opts):
  return obj, position
 
 
-def _get_array(data, position, obj_end, opts):
+def _get_array(data, position, obj_end, opts, element_name):
  """Decode a BSON array to python list."""
  size = _UNPACK_INT(data[position:position + 4])[0]
  end = position + size - 1
@@ -166,12 +173,16 @@ def _get_array(data, position, obj_end, opts):
  element_type = data[position:position + 1]
  # Just skip the keys.
  position = index(b'\x00', position) + 1
- value, position = getter[element_type](data, position, obj_end, opts)
+ try:
+ value, position = getter[element_type](
+ data, position, obj_end, opts, element_name)
+ except KeyError:
+ _raise_unknown_type(element_type, element_name)
  append(value)
  return result, position + 1
 
 
-def _get_binary(data, position, dummy, opts):
+def _get_binary(data, position, dummy0, opts, dummy1):
  """Decode a BSON binary to bson.binary.Binary or python UUID."""
  length, subtype = _UNPACK_LENGTH_SUBTYPE(data[position:position + 5])
  position += 5
@@ -203,19 +214,19 @@ def _get_binary(data, position, dummy, opts):
  return value, end
 
 
-def _get_oid(data, position, dummy0, dummy1):
+def _get_oid(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON ObjectId to bson.objectid.ObjectId."""
  end = position + 12
  return ObjectId(data[position:end]), end
 
 
-def _get_boolean(data, position, dummy0, dummy1):
+def _get_boolean(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON true/false to python True/False."""
  end = position + 1
  return data[position:end] == b"\x01", end
 
 
-def _get_date(data, position, dummy, opts):
+def _get_date(data, position, dummy0, opts, dummy1):
  """Decode a BSON datetime to python datetime.datetime."""
  end = position + 8
  millis = _UNPACK_LONG(data[position:end])[0]
@@ -233,42 +244,44 @@ def _get_date(data, position, dummy, opts):
  return dt, end
 
 
-def _get_code(data, position, obj_end, opts):
+def _get_code(data, position, obj_end, opts, element_name):
  """Decode a BSON code to bson.code.Code."""
- code, position = _get_string(data, position, obj_end, opts)
+ code, position = _get_string(data, position, obj_end, opts, element_name)
  return Code(code), position
 
 
-def _get_code_w_scope(data, position, obj_end, opts):
+def _get_code_w_scope(data, position, obj_end, opts, element_name):
  """Decode a BSON code_w_scope to bson.code.Code."""
- code, position = _get_string(data, position + 4, obj_end, opts)
- scope, position = _get_object(data, position, obj_end, opts)
+ code, position = _get_string(
+ data, position + 4, obj_end, opts, element_name)
+ scope, position = _get_object(data, position, obj_end, opts, element_name)
  return Code(code, scope), position
 
 
-def _get_regex(data, position, dummy0, opts):
+def _get_regex(data, position, dummy0, opts, dummy1):
  """Decode a BSON regex to bson.regex.Regex or a python pattern object."""
  pattern, position = _get_c_string(data, position, opts)
  bson_flags, position = _get_c_string(data, position, opts)
  bson_re = Regex(pattern, bson_flags)
  return bson_re, position
 
 
-def _get_ref(data, position, obj_end, opts):
+def _get_ref(data, position, obj_end, opts, element_name):
  """Decode (deprecated) BSON DBPointer to bson.dbref.DBRef."""
- collection, position = _get_string(data, position, obj_end, opts)
- oid, position = _get_oid(data, position, obj_end, opts)
+ collection, position = _get_string(
+ data, position, obj_end, opts, element_name)
+ oid, position = _get_oid(data, position, obj_end, opts, element_name)
  return DBRef(collection, oid), position
 
 
-def _get_timestamp(data, position, dummy0, dummy1):
+def _get_timestamp(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON timestamp to bson.timestamp.Timestamp."""
  end = position + 8
  inc, timestamp = _UNPACK_TIMESTAMP(data[position:end])
  return Timestamp(timestamp, inc), end
 
 
-def _get_int64(data, position, dummy0, dummy1):
+def _get_int64(data, position, dummy0, dummy1, dummy2):
  """Decode a BSON int64 to bson.int64.Int64."""
  end = position + 8
  return Int64(_UNPACK_LONG(data[position:end])[0]), end
@@ -285,11 +298,11 @@ def _get_int64(data, position, dummy0, dummy1):
  BSONOBJ: _get_object,
  BSONARR: _get_array,
  BSONBIN: _get_binary,
- BSONUND: lambda w, x, y, z: (None, x), # Deprecated undefined
+ BSONUND: lambda v, w, x, y, z: (None, w), # Deprecated undefined
  BSONOID: _get_oid,
  BSONBOO: _get_boolean,
  BSONDAT: _get_date,
- BSONNUL: lambda w, x, y, z: (None, x),
+ BSONNUL: lambda v, w, x, y, z: (None, w),
  BSONRGX: _get_regex,
  BSONREF: _get_ref, # Deprecated DBPointer
  BSONCOD: _get_code,
@@ -298,17 +311,21 @@ def _get_int64(data, position, dummy0, dummy1):
  BSONINT: _get_int,
  BSONTIM: _get_timestamp,
  BSONLON: _get_int64,
- BSONMIN: lambda w, x, y, z: (MinKey(), x),
- BSONMAX: lambda w, x, y, z: (MaxKey(), x)}
+ BSONMIN: lambda v, w, x, y, z: (MinKey(), w),
+ BSONMAX: lambda v, w, x, y, z: (MaxKey(), w)}
 
 
 def _element_to_dict(data, position, obj_end, opts):
  """Decode a single key, value pair."""
  element_type = data[position:position + 1]
  position += 1
  element_name, position = _get_c_string(data, position, opts)
- value, position = _ELEMENT_GETTER[element_type](data,
- position, obj_end, opts)
+ try:
+ value, position = _ELEMENT_GETTER[element_type](data, position,
+ obj_end, opts,
+ element_name)
+ except KeyError:
+ _raise_unknown_type(element_type, element_name)
  return element_name, value, position
 if _USE_C:
  _element_to_dict = _cbson._element_to_dict

bson/_cbsonmodule.c

@@ -1645,7 +1645,7 @@ static PyObject* _cbson_dict_to_bson(PyObject* self, PyObject* args) {
  return result;
 }
 
-static PyObject* get_value(PyObject* self, const char* buffer,
+static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer,
  unsigned* position, unsigned char type,
  unsigned max, const codec_options_t* options) {
  struct module_state *state = GETSTATE(self);
@@ -1815,7 +1815,7 @@ static PyObject* get_value(PyObject* self, const char* buffer,
  Py_DECREF(value);
  goto invalid;
  }
- to_append = get_value(self, buffer, position, bson_type,
+ to_append = get_value(self, name, buffer, position, bson_type,
  max - (unsigned)key_size, options);
  Py_LeaveRecursiveCall();
  if (!to_append) {
@@ -2343,11 +2343,38 @@ static PyObject* get_value(PyObject* self, const char* buffer,
  }
  default:
  {
- PyObject* InvalidDocument = _error("InvalidDocument");
- if (InvalidDocument) {
- PyErr_SetString(InvalidDocument,
- "no c decoder for this type yet");
- Py_DECREF(InvalidDocument);
+ PyObject* InvalidBSON = _error("InvalidBSON");
+ if (InvalidBSON) {
+ PyObject* bobj = PyBytes_FromFormat("%c", type);
+ if (bobj) {
+ PyObject* repr = PyObject_Repr(bobj);
+ Py_DECREF(bobj);
+ /*
+ * See http://bugs.python.org/issue22023 for why we can't
+ * just use PyUnicode_FromFormat with %S or %R to do this
+ * work.
+ */
+ if (repr) {
+ PyObject* left = PyUnicode_FromString(
+ "Detected unknown BSON type ");
+ if (left) {
+ PyObject* lmsg = PyUnicode_Concat(left, repr);
+ Py_DECREF(left);
+ if (lmsg) {
+ PyObject* errmsg = PyUnicode_FromFormat(
+ "%U for fieldname '%U'. Are you using the "
+ "latest driver version?", lmsg, name);
+ if (errmsg) {
+ PyErr_SetObject(InvalidBSON, errmsg);
+ Py_DECREF(errmsg);
+ }
+ Py_DECREF(lmsg);
+ }
+ }
+ Py_DECREF(repr);
+ }
+ }
+ Py_DECREF(InvalidBSON);
  }
  goto invalid;
  }
@@ -2457,10 +2484,10 @@ static int _element_to_dict(PyObject* self, const char* string,
  return -1;
  }
  position += (unsigned)name_length + 1;
- *value = get_value(self, string, &position, type,
+ *value = get_value(self, *name, string, &position, type,
  max - position, options);
  if (!*value) {
- Py_DECREF(name);
+ Py_DECREF(*name);
  return -1;
  }
  return position;

test/test_bson.py

@@ -401,6 +401,24 @@ def test_basic_encode(self):
  b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x00"
  b"\x00")
 
+ def test_unknown_type(self):
+ # Repr value differs with major python version
+ part = "type %r for fieldname 'foo'" % (b'\x13',)
+ docs = [
+ b'\x0e\x00\x00\x00\x13foo\x00\x01\x00\x00\x00\x00',
+ (b'\x16\x00\x00\x00\x04foo\x00\x0c\x00\x00\x00\x130'
+ b'\x00\x01\x00\x00\x00\x00\x00'),
+ (b' \x00\x00\x00\x04bar\x00\x16\x00\x00\x00\x030\x00\x0e\x00\x00'
+ b'\x00\x13foo\x00\x01\x00\x00\x00\x00\x00\x00')]
+ for bs in docs:
+ try:
+ bson.BSON(bs).decode()
+ except Exception as exc:
+ self.assertTrue(isinstance(exc, InvalidBSON))
+ self.assertTrue(part in str(exc))
+ else:
+ self.fail("Failed to raise an exception.")
+
  def test_dbpointer(self):
  # *Note* - DBPointer and DBRef are *not* the same thing. DBPointer
  # is a deprecated BSON type. DBRef is a convention that does not