bpo-38324: Fix test__locale.py Windows failures (pythonGH-20529)

Use wide-char _W_* fields of lconv structure on Windows Remove "ps_AF" from test__locale.known_numerics on Windows

Author: TIGirardi

Lib/test/test__locale.py

@@ -72,6 +72,10 @@ def accept(loc):
  'ps_AF': ('\u066b', '\u066c'),
 }
 
+if sys.platform == 'win32':
+ # ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
+ del known_numerics['ps_AF']
+
 class _LocaleTests(unittest.TestCase):
 
  def setUp(self):

Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst

@@ -0,0 +1 @@
+Avoid Unicode errors when accessing certain locale data on Windows.

Modules/_localemodule.c

@@ -155,6 +155,7 @@ locale_is_ascii(const char *str)
 static int
 locale_decode_monetary(PyObject *dict, struct lconv *lc)
 {
+#ifndef MS_WINDOWS
  int change_locale;
  change_locale = (!locale_is_ascii(lc->int_curr_symbol)
  || !locale_is_ascii(lc->currency_symbol)
@@ -190,12 +191,18 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
  }
  }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
  int res = -1;
 
 #define RESULT_STRING(ATTR) \
  do { \
  PyObject *obj; \
- obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \
+ obj = GET_LOCALE_STRING(ATTR); \
  if (obj == NULL) { \
  goto done; \
  } \
@@ -211,14 +218,17 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
  RESULT_STRING(mon_decimal_point);
  RESULT_STRING(mon_thousands_sep);
 #undef RESULT_STRING
+#undef GET_LOCALE_STRING
 
  res = 0;
 
 done:
+#ifndef MS_WINDOWS
  if (loc != NULL) {
  setlocale(LC_CTYPE, oldloc);
  }
  PyMem_Free(oldloc);
+#endif
  return res;
 }
 
@@ -258,9 +268,15 @@ _locale_localeconv_impl(PyObject *module)
  Py_DECREF(obj); \
  } while (0)
 
+#ifdef MS_WINDOWS
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#else
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#endif
 #define RESULT_STRING(s)\
  do { \
- x = PyUnicode_DecodeLocale(lc->s, NULL); \
+ x = GET_LOCALE_STRING(s); \
  RESULT(#s, x); \
  } while (0)
 
@@ -289,8 +305,10 @@ _locale_localeconv_impl(PyObject *module)
  RESULT_INT(n_sign_posn);
 
  /* Numeric information: LC_NUMERIC encoding */
- PyObject *decimal_point, *thousands_sep;
+ PyObject *decimal_point = NULL, *thousands_sep = NULL;
  if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) {
+ Py_XDECREF(decimal_point);
+ Py_XDECREF(thousands_sep);
  goto failed;
  }
 
@@ -319,6 +337,7 @@ _locale_localeconv_impl(PyObject *module)
 #undef RESULT
 #undef RESULT_STRING
 #undef RESULT_INT
+#undef GET_LOCALE_STRING
 }
 
 #if defined(HAVE_WCSCOLL)

Python/fileutils.c

@@ -2047,6 +2047,7 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
  assert(decimal_point != NULL);
  assert(thousands_sep != NULL);
 
+#ifndef MS_WINDOWS
  int change_locale = 0;
  if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
  change_locale = 1;
@@ -2085,26 +2086,36 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
  }
  }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows strcut lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
  int res = -1;
 
- *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
+ *decimal_point = GET_LOCALE_STRING(decimal_point);
  if (*decimal_point == NULL) {
  goto done;
  }
 
- *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
+ *thousands_sep = GET_LOCALE_STRING(thousands_sep);
  if (*thousands_sep == NULL) {
  goto done;
  }
 
  res = 0;
 
 done:
+#ifndef MS_WINDOWS
  if (loc != NULL) {
  setlocale(LC_CTYPE, oldloc);
  }
  PyMem_Free(oldloc);
+#endif
  return res;
+
+#undef GET_LOCALE_STRING
 }
 
 /* Our selection logic for which function to use is as follows: