bpo-28180: Standard stream & FS encoding differ on Mac OS X (GH-2208)

In the C locale on Mac OS X, the default filesystem encoding used for operating system interfaces is UTF-8, but the default encoding used on the standard streams is still ASCII. Setting the POSIX locale also behaves differently from setting other locales on Mac OS X, so skip that in the test suite for now.

Author: ncoghlan

Lib/test/test_c_locale_coercion.py

@@ -16,10 +16,11 @@
 
 # Set our expectation for the default encoding used in the C locale
 # for the filesystem encoding and the standard streams
+C_LOCALE_STREAM_ENCODING = "ascii"
 if sys.platform == "darwin":
- EXPECTED_C_LOCALE_FSENCODING = "utf-8"
+ C_LOCALE_FS_ENCODING = "utf-8"
 else:
- EXPECTED_C_LOCALE_FSENCODING = "ascii"
+ C_LOCALE_FS_ENCODING = C_LOCALE_STREAM_ENCODING
 
 # XXX (ncoghlan): The above is probably still wrong for:
 # * Windows when PYTHONLEGACYWINDOWSFSENCODING is set
@@ -52,15 +53,15 @@ class EncodingDetails(_EncodingDetails):
  ])
 
  @classmethod
- def get_expected_details(cls, expected_fsencoding):
+ def get_expected_details(cls, fs_encoding, stream_encoding):
  """Returns expected child process details for a given encoding"""
- _stream = expected_fsencoding + ":{}"
+ _stream = stream_encoding + ":{}"
  # stdin and stdout should use surrogateescape either because the
  # coercion triggered, or because the C locale was detected
  stream_info = 2*[_stream.format("surrogateescape")]
  # stderr should always use backslashreplace
  stream_info.append(_stream.format("backslashreplace"))
- return dict(cls(expected_fsencoding, *stream_info)._asdict())
+ return dict(cls(fs_encoding, *stream_info)._asdict())
 
  @staticmethod
  def _handle_output_variations(data):
@@ -107,21 +108,22 @@ class _ChildProcessEncodingTestCase(unittest.TestCase):
 
  def _check_child_encoding_details(self,
  env_vars,
- expected_fsencoding,
+ expected_fs_encoding,
+ expected_stream_encoding,
  expected_warning):
  """Check the C locale handling for the given process environment
 
  Parameters:
- expected_fsencoding: the encoding the child is expected to report
- allow_c_locale: setting to use for PYTHONALLOWCLOCALE
- None: don't set the variable at all
- str: the value set in the child's environment
+ expected_fs_encoding: expected sys.getfilesystemencoding() result
+ expected_stream_encoding: expected encoding for standard streams
+ expected_warning: stderr output to expect (if any)
  """
  result = EncodingDetails.get_child_details(env_vars)
  encoding_details, stderr_lines = result
  self.assertEqual(encoding_details,
  EncodingDetails.get_expected_details(
- expected_fsencoding))
+ expected_fs_encoding,
+ expected_stream_encoding))
  self.assertEqual(stderr_lines, expected_warning)
 
 # Details of the shared library warning emitted at runtime
@@ -140,12 +142,17 @@ class LocaleWarningTests(_ChildProcessEncodingTestCase):
  def test_library_c_locale_warning(self):
  self.maxDiff = None
  for locale_to_set in ("C", "POSIX", "invalid.ascii"):
+ # XXX (ncoghlan): Mac OS X doesn't behave as expected in the
+ # POSIX locale, so we skip that for now
+ if sys.platform == "darwin" and locale_to_set == "POSIX":
+ continue
  var_dict = {
  "LC_ALL": locale_to_set
  }
  with self.subTest(forced_locale=locale_to_set):
  self._check_child_encoding_details(var_dict,
- EXPECTED_C_LOCALE_FSENCODING,
+ C_LOCALE_FS_ENCODING,
+ C_LOCALE_STREAM_ENCODING,
  [LIBRARY_C_LOCALE_WARNING])
 
 # Details of the CLI locale coercion warning emitted at runtime
@@ -190,7 +197,8 @@ def test_external_target_locale_configuration(self):
  self.maxDiff = None
 
  expected_warning = []
- expected_fsencoding = "utf-8"
+ expected_fs_encoding = "utf-8"
+ expected_stream_encoding = "utf-8"
 
  base_var_dict = {
  "LANG": "",
@@ -209,7 +217,8 @@ def test_external_target_locale_configuration(self):
  var_dict = base_var_dict.copy()
  var_dict[env_var] = locale_to_set
  self._check_child_encoding_details(var_dict,
- expected_fsencoding,
+ expected_fs_encoding,
+ expected_stream_encoding,
  expected_warning)
 
 
@@ -220,12 +229,13 @@ def test_external_target_locale_configuration(self):
 class LocaleCoercionTests(_LocaleCoercionTargetsTestCase):
  # Test implicit reconfiguration of the environment during CLI startup
 
- def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
+ def _check_c_locale_coercion(self, fs_encoding, stream_encoding, coerce_c_locale):
  """Check the C locale handling for various configurations
 
  Parameters:
- expected_fsencoding: the encoding the child is expected to report
- allow_c_locale: setting to use for PYTHONALLOWCLOCALE
+ fs_encoding: expected sys.getfilesystemencoding() result
+ stream_encoding: expected encoding for standard streams
+ coerce_c_locale: setting to use for PYTHONCOERCECLOCALE
  None: don't set the variable at all
  str: the value set in the child's environment
  """
@@ -246,6 +256,10 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
  }
  for env_var in ("LANG", "LC_CTYPE"):
  for locale_to_set in ("", "C", "POSIX", "invalid.ascii"):
+ # XXX (ncoghlan): Mac OS X doesn't behave as expected in the
+ # POSIX locale, so we skip that for now
+ if sys.platform == "darwin" and locale_to_set == "POSIX":
+ continue
  with self.subTest(env_var=env_var,
  nominal_locale=locale_to_set,
  PYTHONCOERCECLOCALE=coerce_c_locale):
@@ -254,22 +268,24 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
  if coerce_c_locale is not None:
  var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
  self._check_child_encoding_details(var_dict,
- expected_fsencoding,
+ fs_encoding,
+ stream_encoding,
  expected_warning)
 
  def test_test_PYTHONCOERCECLOCALE_not_set(self):
  # This should coerce to the first available target locale by default
- self._check_c_locale_coercion("utf-8", coerce_c_locale=None)
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=None)
 
  def test_PYTHONCOERCECLOCALE_not_zero(self):
  # *Any* string other that "0" is considered "set" for our purposes
  # and hence should result in the locale coercion being enabled
  for setting in ("", "1", "true", "false"):
- self._check_c_locale_coercion("utf-8", coerce_c_locale=setting)
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=setting)
 
  def test_PYTHONCOERCECLOCALE_set_to_zero(self):
  # The setting "0" should result in the locale coercion being disabled
- self._check_c_locale_coercion(EXPECTED_C_LOCALE_FSENCODING,
+ self._check_c_locale_coercion(C_LOCALE_FS_ENCODING,
+ C_LOCALE_STREAM_ENCODING,
  coerce_c_locale="0")