Mailing List Archive

https://github.com/python/cpython/commit/4cde523aa467c92a713674c793ab87e7c56486c4
commit: 4cde523aa467c92a713674c793ab87e7c56486c4
branch: 3.8
author: Miss Skeleton (bot) <31488909+miss-islington@users.noreply.github.com>
committer: GitHub <noreply@github.com>
date: 2020-10-20T04:58:06-07:00
summary:

bpo-38324: Fix test__locale.py Windows failures (GH-20529)


Use wide-char _W_* fields of lconv structure on Windows
Remove "ps_AF" from test__locale.known_numerics on Windows
(cherry picked from commit f2312037e3a974d26ed3e23884f94c6af111a27a)

Co-authored-by: TIGirardi <tiagoigirardi@gmail.com>

files:
A Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
M Lib/test/test__locale.py
M Modules/_localemodule.c
M Python/fileutils.c

diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index ab4e24796188c..560f077274275 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -72,6 +72,10 @@ def accept(loc):
'ps_AF': ('\u066b', '\u066c'),
}

+if sys.platform == 'win32':
+ # ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
+ del known_numerics['ps_AF']
+
class _LocaleTests(unittest.TestCase):

def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
new file mode 100644
index 0000000000000..c45aa13091429
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
@@ -0,0 +1 @@
+Avoid Unicode errors when accessing certain locale data on Windows.
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index 036bdb301f320..d54f70904e456 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -131,6 +131,7 @@ locale_is_ascii(const char *str)
static int
locale_decode_monetary(PyObject *dict, struct lconv *lc)
{
+#ifndef MS_WINDOWS
int change_locale;
change_locale = (!locale_is_ascii(lc->int_curr_symbol)
|| !locale_is_ascii(lc->currency_symbol)
@@ -166,12 +167,18 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
}
}

+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
int res = -1;

#define RESULT_STRING(ATTR) \
do { \
PyObject *obj; \
- obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \
+ obj = GET_LOCALE_STRING(ATTR); \
if (obj == NULL) { \
goto done; \
} \
@@ -187,14 +194,17 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
RESULT_STRING(mon_decimal_point);
RESULT_STRING(mon_thousands_sep);
#undef RESULT_STRING
+#undef GET_LOCALE_STRING

res = 0;

done:
+#ifndef MS_WINDOWS
if (loc != NULL) {
setlocale(LC_CTYPE, oldloc);
}
PyMem_Free(oldloc);
+#endif
return res;
}

@@ -230,9 +240,15 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
Py_DECREF(obj); \
} while (0)

+#ifdef MS_WINDOWS
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#else
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#endif
#define RESULT_STRING(s)\
do { \
- x = PyUnicode_DecodeLocale(lc->s, NULL); \
+ x = GET_LOCALE_STRING(s); \
RESULT(#s, x); \
} while (0)

@@ -261,8 +277,10 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
RESULT_INT(n_sign_posn);

/* Numeric information: LC_NUMERIC encoding */
- PyObject *decimal_point, *thousands_sep;
+ PyObject *decimal_point = NULL, *thousands_sep = NULL;
if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) {
+ Py_XDECREF(decimal_point);
+ Py_XDECREF(thousands_sep);
goto failed;
}

@@ -291,6 +309,7 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
#undef RESULT
#undef RESULT_STRING
#undef RESULT_INT
+#undef GET_LOCALE_STRING
}

#if defined(HAVE_WCSCOLL)
diff --git a/Python/fileutils.c b/Python/fileutils.c
index b274116745efe..25516c23bf7ab 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -1933,6 +1933,7 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
assert(decimal_point != NULL);
assert(thousands_sep != NULL);

+#ifndef MS_WINDOWS
int change_locale = 0;
if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
change_locale = 1;
@@ -1971,14 +1972,20 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
}
}

+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows strcut lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
int res = -1;

- *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
+ *decimal_point = GET_LOCALE_STRING(decimal_point);
if (*decimal_point == NULL) {
goto done;
}

- *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
+ *thousands_sep = GET_LOCALE_STRING(thousands_sep);
if (*thousands_sep == NULL) {
goto done;
}
@@ -1986,9 +1993,13 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
res = 0;

done:
+#ifndef MS_WINDOWS
if (loc != NULL) {
setlocale(LC_CTYPE, oldloc);
}
PyMem_Free(oldloc);
+#endif
return res;
+
+#undef GET_LOCALE_STRING
}

_______________________________________________
Python-checkins mailing list
Python-checkins@python.org
https://mail.python.org/mailman/listinfo/python-checkins

https://github.com/python/cpython/commit/c17ff5cad2952d9abd63aeae060cae5658257b0c
commit: c17ff5cad2952d9abd63aeae060cae5658257b0c
branch: 3.9
author: Miss Skeleton (bot) <31488909+miss-islington@users.noreply.github.com>
committer: GitHub <noreply@github.com>
date: 2020-10-20T05:07:14-07:00
summary:

bpo-38324: Fix test__locale.py Windows failures (GH-20529)


Use wide-char _W_* fields of lconv structure on Windows
Remove "ps_AF" from test__locale.known_numerics on Windows
(cherry picked from commit f2312037e3a974d26ed3e23884f94c6af111a27a)

Co-authored-by: TIGirardi <tiagoigirardi@gmail.com>

files:
A Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
M Lib/test/test__locale.py
M Modules/_localemodule.c
M Python/fileutils.c

diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index cda0ee91b700f..59a00bad7d98a 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -72,6 +72,10 @@ def accept(loc):
'ps_AF': ('\u066b', '\u066c'),
}

+if sys.platform == 'win32':
+ # ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
+ del known_numerics['ps_AF']
+
class _LocaleTests(unittest.TestCase):

def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
new file mode 100644
index 0000000000000..c45aa13091429
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
@@ -0,0 +1 @@
+Avoid Unicode errors when accessing certain locale data on Windows.
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index 0819d0e192408..2e353bba00bf3 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -144,6 +144,7 @@ locale_is_ascii(const char *str)
static int
locale_decode_monetary(PyObject *dict, struct lconv *lc)
{
+#ifndef MS_WINDOWS
int change_locale;
change_locale = (!locale_is_ascii(lc->int_curr_symbol)
|| !locale_is_ascii(lc->currency_symbol)
@@ -179,12 +180,18 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
}
}

+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
int res = -1;

#define RESULT_STRING(ATTR) \
do { \
PyObject *obj; \
- obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \
+ obj = GET_LOCALE_STRING(ATTR); \
if (obj == NULL) { \
goto done; \
} \
@@ -200,14 +207,17 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
RESULT_STRING(mon_decimal_point);
RESULT_STRING(mon_thousands_sep);
#undef RESULT_STRING
+#undef GET_LOCALE_STRING

res = 0;

done:
+#ifndef MS_WINDOWS
if (loc != NULL) {
setlocale(LC_CTYPE, oldloc);
}
PyMem_Free(oldloc);
+#endif
return res;
}

@@ -243,9 +253,15 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
Py_DECREF(obj); \
} while (0)

+#ifdef MS_WINDOWS
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#else
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#endif
#define RESULT_STRING(s)\
do { \
- x = PyUnicode_DecodeLocale(lc->s, NULL); \
+ x = GET_LOCALE_STRING(s); \
RESULT(#s, x); \
} while (0)

@@ -274,8 +290,10 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
RESULT_INT(n_sign_posn);

/* Numeric information: LC_NUMERIC encoding */
- PyObject *decimal_point, *thousands_sep;
+ PyObject *decimal_point = NULL, *thousands_sep = NULL;
if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) {
+ Py_XDECREF(decimal_point);
+ Py_XDECREF(thousands_sep);
goto failed;
}

@@ -304,6 +322,7 @@ PyLocale_localeconv(PyObject* self, PyObject *Py_UNUSED(ignored))
#undef RESULT
#undef RESULT_STRING
#undef RESULT_INT
+#undef GET_LOCALE_STRING
}

#if defined(HAVE_WCSCOLL)
diff --git a/Python/fileutils.c b/Python/fileutils.c
index 2c86828ba989a..397ac34e8073d 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -2032,6 +2032,7 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
assert(decimal_point != NULL);
assert(thousands_sep != NULL);

+#ifndef MS_WINDOWS
int change_locale = 0;
if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
change_locale = 1;
@@ -2070,14 +2071,20 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
}
}

+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows strcut lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
int res = -1;

- *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
+ *decimal_point = GET_LOCALE_STRING(decimal_point);
if (*decimal_point == NULL) {
goto done;
}

- *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
+ *thousands_sep = GET_LOCALE_STRING(thousands_sep);
if (*thousands_sep == NULL) {
goto done;
}
@@ -2085,9 +2092,13 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
res = 0;

done:
+#ifndef MS_WINDOWS
if (loc != NULL) {
setlocale(LC_CTYPE, oldloc);
}
PyMem_Free(oldloc);
+#endif
return res;
+
+#undef GET_LOCALE_STRING
}

_______________________________________________
Python-checkins mailing list
Python-checkins@python.org
https://mail.python.org/mailman/listinfo/python-checkins