https://github.com/python/cpython/commit/e05d202ebf0904cf0a980fde76f4a38bb8dc7c3b
commit: e05d202ebf0904cf0a980fde76f4a38bb8dc7c3b
branch: main
author: Victor Stinner <vstinner@python.org>
committer: vstinner <vstinner@python.org>
date: 2024-04-16T22:05:12+02:00
summary:
gh-117755: Remove tests on huge memory allocations (#117938)
Remove unreliable tests on huge memory allocations:
* Remove test_maxcontext_exact_arith() of test_decimal.
Stefan Krah, test author, agreed on removing the test:
https://github.com/python/cpython/issues/114331#issuecomment-1925731273
* Remove test_constructor() tests of test_io.
Sam Gross suggests remove them:
https://github.com/python/cpython/pull/117809#pullrequestreview-2003889558
On Linux, depending how overcommit is configured, especially on Linux
s390x, a huge memory allocation (half or more of the full address
space) can succeed, but then the process will eat the full system
swap and make the system slower and slower until the whole system
becomes unusable.
Moreover, these tests had to be skipped when Python is built with
sanitizers.
files:
M Lib/test/test_decimal.py
M Lib/test/test_io.py
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index 05dcb25a7e5950..7010c34792e093 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -34,12 +34,10 @@
import locale
from test.support import (is_resource_enabled,
requires_IEEE_754, requires_docstrings,
- check_sanitizer,
check_disallow_instantiation)
from test.support import (TestFailed,
run_with_locale, cpython_only,
- darwin_malloc_err_warning, is_emscripten,
- skip_on_s390x)
+ darwin_malloc_err_warning)
from test.support.import_helper import import_fresh_module
from test.support import threading_helper
from test.support import warnings_helper
@@ -5644,50 +5642,6 @@ def __abs__(self):
self.assertEqual(Decimal.from_float(cls(101.1)),
Decimal.from_float(101.1))
- # Issue 41540:
- @unittest.skipIf(sys.platform.startswith("aix"),
- "AIX: default ulimit: test is flaky because of extreme over-allocation")
- @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten")
- @unittest.skipIf(check_sanitizer(address=True, memory=True),
- "ASAN/MSAN sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-114331: The test allocates 784 271 641 GiB and mimalloc does not fail
- # to allocate it when using mimalloc on s390x.
- @skip_on_s390x
- def test_maxcontext_exact_arith(self):
-
- # Make sure that exact operations do not raise MemoryError due
- # to huge intermediate values when the context precision is very
- # large.
-
- # The following functions fill the available precision and are
- # therefore not suitable for large precisions (by design of the
- # specification).
- MaxContextSkip = [.'logical_invert', 'next_minus', 'next_plus',
- 'logical_and', 'logical_or', 'logical_xor',
- 'next_toward', 'rotate', 'shift']
-
- Decimal = C.Decimal
- Context = C.Context
- localcontext = C.localcontext
-
- # Here only some functions that are likely candidates for triggering a
- # MemoryError are tested. deccheck.py has an exhaustive test.
- maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX)
- with localcontext(maxcontext):
- self.assertEqual(Decimal(0).exp(), 1)
- self.assertEqual(Decimal(1).ln(), 0)
- self.assertEqual(Decimal(1).log10(), 0)
- self.assertEqual(Decimal(10**2).log10(), 2)
- self.assertEqual(Decimal(10**223).log10(), 223)
- self.assertEqual(Decimal(10**19).logb(), 19)
- self.assertEqual(Decimal(4).sqrt(), 2)
- self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5'))
- self.assertEqual(divmod(Decimal(10), 3), (3, 1))
- self.assertEqual(Decimal(10) // 3, 3)
- self.assertEqual(Decimal(4) / 2, 2)
- self.assertEqual(Decimal(400) ** -1, Decimal('0.0025'))
-
def test_c_immutable_types(self):
SignalDict = type(C.Context().flags)
SignalDictMixin = SignalDict.__bases__[0]
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index c3dc0572c58c27..e5cb08c2cdd04c 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -40,8 +40,7 @@
from test.support.script_helper import (
assert_python_ok, assert_python_failure, run_python_until_end)
from test.support import (
- import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper,
- skip_on_s390x
+ import_helper, is_apple, os_helper, threading_helper, warnings_helper,
)
from test.support.os_helper import FakePath
@@ -1698,22 +1697,6 @@ def test_seek_character_device_file(self):
class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
tp = io.BufferedReader
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedReaderTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_initialization(self):
rawio = self.MockRawIO([b"abc"])
bufio = self.tp(rawio)
@@ -2069,22 +2052,6 @@ def test_slow_close_from_thread(self):
class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
tp = io.BufferedWriter
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedWriterTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_initialization(self):
rawio = self.MockRawIO()
bufio = self.tp(rawio)
@@ -2594,22 +2561,6 @@ def test_interleaved_readline_write(self):
class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
tp = io.BufferedRandom
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedRandomTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_garbage_collection(self):
CBufferedReaderTest.test_garbage_collection(self)
CBufferedWriterTest.test_garbage_collection(self)
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-leave@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: list-python-checkins@lists.gossamer-threads.com
commit: e05d202ebf0904cf0a980fde76f4a38bb8dc7c3b
branch: main
author: Victor Stinner <vstinner@python.org>
committer: vstinner <vstinner@python.org>
date: 2024-04-16T22:05:12+02:00
summary:
gh-117755: Remove tests on huge memory allocations (#117938)
Remove unreliable tests on huge memory allocations:
* Remove test_maxcontext_exact_arith() of test_decimal.
Stefan Krah, test author, agreed on removing the test:
https://github.com/python/cpython/issues/114331#issuecomment-1925731273
* Remove test_constructor() tests of test_io.
Sam Gross suggests remove them:
https://github.com/python/cpython/pull/117809#pullrequestreview-2003889558
On Linux, depending how overcommit is configured, especially on Linux
s390x, a huge memory allocation (half or more of the full address
space) can succeed, but then the process will eat the full system
swap and make the system slower and slower until the whole system
becomes unusable.
Moreover, these tests had to be skipped when Python is built with
sanitizers.
files:
M Lib/test/test_decimal.py
M Lib/test/test_io.py
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index 05dcb25a7e5950..7010c34792e093 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -34,12 +34,10 @@
import locale
from test.support import (is_resource_enabled,
requires_IEEE_754, requires_docstrings,
- check_sanitizer,
check_disallow_instantiation)
from test.support import (TestFailed,
run_with_locale, cpython_only,
- darwin_malloc_err_warning, is_emscripten,
- skip_on_s390x)
+ darwin_malloc_err_warning)
from test.support.import_helper import import_fresh_module
from test.support import threading_helper
from test.support import warnings_helper
@@ -5644,50 +5642,6 @@ def __abs__(self):
self.assertEqual(Decimal.from_float(cls(101.1)),
Decimal.from_float(101.1))
- # Issue 41540:
- @unittest.skipIf(sys.platform.startswith("aix"),
- "AIX: default ulimit: test is flaky because of extreme over-allocation")
- @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten")
- @unittest.skipIf(check_sanitizer(address=True, memory=True),
- "ASAN/MSAN sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-114331: The test allocates 784 271 641 GiB and mimalloc does not fail
- # to allocate it when using mimalloc on s390x.
- @skip_on_s390x
- def test_maxcontext_exact_arith(self):
-
- # Make sure that exact operations do not raise MemoryError due
- # to huge intermediate values when the context precision is very
- # large.
-
- # The following functions fill the available precision and are
- # therefore not suitable for large precisions (by design of the
- # specification).
- MaxContextSkip = [.'logical_invert', 'next_minus', 'next_plus',
- 'logical_and', 'logical_or', 'logical_xor',
- 'next_toward', 'rotate', 'shift']
-
- Decimal = C.Decimal
- Context = C.Context
- localcontext = C.localcontext
-
- # Here only some functions that are likely candidates for triggering a
- # MemoryError are tested. deccheck.py has an exhaustive test.
- maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX)
- with localcontext(maxcontext):
- self.assertEqual(Decimal(0).exp(), 1)
- self.assertEqual(Decimal(1).ln(), 0)
- self.assertEqual(Decimal(1).log10(), 0)
- self.assertEqual(Decimal(10**2).log10(), 2)
- self.assertEqual(Decimal(10**223).log10(), 223)
- self.assertEqual(Decimal(10**19).logb(), 19)
- self.assertEqual(Decimal(4).sqrt(), 2)
- self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5'))
- self.assertEqual(divmod(Decimal(10), 3), (3, 1))
- self.assertEqual(Decimal(10) // 3, 3)
- self.assertEqual(Decimal(4) / 2, 2)
- self.assertEqual(Decimal(400) ** -1, Decimal('0.0025'))
-
def test_c_immutable_types(self):
SignalDict = type(C.Context().flags)
SignalDictMixin = SignalDict.__bases__[0]
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index c3dc0572c58c27..e5cb08c2cdd04c 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -40,8 +40,7 @@
from test.support.script_helper import (
assert_python_ok, assert_python_failure, run_python_until_end)
from test.support import (
- import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper,
- skip_on_s390x
+ import_helper, is_apple, os_helper, threading_helper, warnings_helper,
)
from test.support.os_helper import FakePath
@@ -1698,22 +1697,6 @@ def test_seek_character_device_file(self):
class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
tp = io.BufferedReader
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedReaderTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_initialization(self):
rawio = self.MockRawIO([b"abc"])
bufio = self.tp(rawio)
@@ -2069,22 +2052,6 @@ def test_slow_close_from_thread(self):
class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
tp = io.BufferedWriter
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedWriterTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_initialization(self):
rawio = self.MockRawIO()
bufio = self.tp(rawio)
@@ -2594,22 +2561,6 @@ def test_interleaved_readline_write(self):
class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
tp = io.BufferedRandom
- @skip_if_sanitizer(memory=True, address=True, thread=True,
- reason="sanitizer defaults to crashing "
- "instead of returning NULL for malloc failure.")
- # gh-117755: The test allocates 9 223 372 036 854 775 807 bytes
- # (0x7fffffffffffffff) and mimalloc fails with a division by zero on s390x.
- @skip_on_s390x
- def test_constructor(self):
- BufferedRandomTest.test_constructor(self)
- # The allocation can succeed on 32-bit builds, e.g. with more
- # than 2 GiB RAM and a 64-bit kernel.
- if sys.maxsize > 0x7FFFFFFF:
- rawio = self.MockRawIO()
- bufio = self.tp(rawio)
- self.assertRaises((OverflowError, MemoryError, ValueError),
- bufio.__init__, rawio, sys.maxsize)
-
def test_garbage_collection(self):
CBufferedReaderTest.test_garbage_collection(self)
CBufferedWriterTest.test_garbage_collection(self)
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-leave@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: list-python-checkins@lists.gossamer-threads.com