Move time consuming utf8 tests out of the main suite

Author: behackett

test/test_bson.py

@@ -650,69 +650,6 @@ def test_utf8(self):
  z = {iso8859_bytes: "hello"}
  self.assertRaises(InvalidStringData, BSON.encode, z)
 
- # Verify that python and bson have the same understanding of
- # legal utf-8 if the first byte is 0xf4 (244)
- def _assert_same_utf8_validation(self, data):
- try:
- data.decode('utf-8')
- py_is_legal = True
- except UnicodeDecodeError:
- py_is_legal = False
-
- try:
- BSON.encode({'x': data})
- bson_is_legal = True
- except InvalidStringData:
- bson_is_legal = False
-
- self.assertEqual(py_is_legal, bson_is_legal, data)
-
- @unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
- def test_legal_utf8_full_coverage(self):
- # this tests takes 400 seconds. Which is too long to run each time.
- # However it is the only one which covers all possible bit combinations
- # in the 244 space.
- b1 = chr(0xf4)
-
- for b2 in map(chr, range(255)):
- m2 = b1 + b2
- self._assert_same_utf8_validation(m2)
-
- for b3 in map(chr, range(255)):
- m3 = m2 + b3
- self._assert_same_utf8_validation(m3)
-
- for b4 in map(chr, range(255)):
- m4 = m3 + b4
- self._assert_same_utf8_validation(m4)
-
- # In python3:
- # - 'bytes' are not checked with isLegalutf
- # - 'unicode' I cannot create unicode objects with invalid utf8, since it
- # would result in non valid code-points.
- @unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
- def test_legal_utf8_few_samples(self):
- good_samples = [
- '\xf4\x80\x80\x80',
- '\xf4\x8a\x80\x80',
- '\xf4\x8e\x80\x80',
- '\xf4\x81\x80\x80',
- ]
-
- for data in good_samples:
- self._assert_same_utf8_validation(data)
-
- bad_samples = [
- '\xf4\x00\x80\x80',
- '\xf4\x3a\x80\x80',
- '\xf4\x7f\x80\x80',
- '\xf4\x90\x80\x80',
- '\xf4\xff\x80\x80',
- ]
-
- for data in bad_samples:
- self._assert_same_utf8_validation(data)
-
  def test_null_character(self):
  doc = {"a": "\x00"}
  self.assertEqual(doc, BSON.encode(doc).decode())

test/unicode/test_utf8.py

@@ -0,0 +1,76 @@
+import sys
+
+sys.path[0:0] = [""]
+
+from bson import BSON
+from bson.errors import InvalidStringData
+from bson.py3compat import PY3
+from test import unittest
+
+class TestUTF8(unittest.TestCase):
+
+ # Verify that python and bson have the same understanding of
+ # legal utf-8 if the first byte is 0xf4 (244)
+ def _assert_same_utf8_validation(self, data):
+ try:
+ data.decode('utf-8')
+ py_is_legal = True 
+ except UnicodeDecodeError:
+ py_is_legal = False
+
+ try:
+ BSON.encode({'x': data})
+ bson_is_legal = True 
+ except InvalidStringData:
+ bson_is_legal = False
+
+ self.assertEqual(py_is_legal, bson_is_legal, data)
+
+ @unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
+ def test_legal_utf8_full_coverage(self):
+ # This test takes 400 seconds. Which is too long to run each time.
+ # However it is the only one which covers all possible bit combinations
+ # in the 244 space.
+ b1 = chr(0xf4)
+
+ for b2 in map(chr, range(255)):
+ m2 = b1 + b2 
+ self._assert_same_utf8_validation(m2)
+
+ for b3 in map(chr, range(255)):
+ m3 = m2 + b3 
+ self._assert_same_utf8_validation(m3)
+
+ for b4 in map(chr, range(255)):
+ m4 = m3 + b4 
+ self._assert_same_utf8_validation(m4)
+
+ # In python3:
+ # - 'bytes' are not checked with isLegalutf
+ # - 'unicode' We cannot create unicode objects with invalid utf8, since it
+ # would result in non valid code-points.
+ @unittest.skipIf(PY3, "python3 has strong separation between bytes/unicode")
+ def test_legal_utf8_few_samples(self):
+ good_samples = [
+ '\xf4\x80\x80\x80',
+ '\xf4\x8a\x80\x80',
+ '\xf4\x8e\x80\x80',
+ '\xf4\x81\x80\x80',
+ ]
+
+ for data in good_samples:
+ self._assert_same_utf8_validation(data)
+
+ bad_samples = [
+ '\xf4\x00\x80\x80',
+ '\xf4\x3a\x80\x80',
+ '\xf4\x7f\x80\x80',
+ '\xf4\x90\x80\x80',
+ '\xf4\xff\x80\x80',
+ ]
+
+ for data in bad_samples:
+ self._assert_same_utf8_validation(data)
+
+if __name__ == "__main__":
+ unittest.main()