diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py
index e76f79c274e..ad9a9ea8303 100644
--- a/Lib/test/list_tests.py
+++ b/Lib/test/list_tests.py
@@ -6,7 +6,7 @@
from functools import cmp_to_key
from test import seq_tests
-from test.support import ALWAYS_EQ, NEVER_EQ
+from test.support import ALWAYS_EQ, NEVER_EQ, run_with_limited_c_stack
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
@@ -60,6 +60,7 @@ def test_repr(self):
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
+ @run_with_limited_c_stack(200_000)
@skip_wasi_stack_overflow()
@skip_emscripten_stack_overflow()
def test_repr_deep(self):
diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py
index 20306e1526d..1358200add5 100644
--- a/Lib/test/mapping_tests.py
+++ b/Lib/test/mapping_tests.py
@@ -622,6 +622,7 @@ def __repr__(self):
d = self._full_mapping({1: BadRepr()})
self.assertRaises(Exc, repr, d)
+ @support.run_with_limited_c_stack()
@support.skip_wasi_stack_overflow()
@support.skip_emscripten_stack_overflow()
@support.skip_if_sanitizer("requires deep stack", ub=True)
diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py
index ca93e99d142..50ccf66b8da 100644
--- a/Lib/test/test_calendar.py
+++ b/Lib/test/test_calendar.py
@@ -509,6 +509,11 @@ def test_deprecation_warning(self):
"The 'January' attribute is deprecated, use 'JANUARY' instead"
):
calendar.January
+ with self.assertWarnsRegex(
+ DeprecationWarning,
+ "The 'February' attribute is deprecated, use 'FEBRUARY' instead"
+ ):
+ calendar.February
def test_isleap(self):
# Make sure that the return is right for a few years, and
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index 2e1568d5ea2..248041fdf19 100644
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -4,43 +4,64 @@
"""
import unittest
import warnings
-from test.support import warnings_helper
+from test.support import subTests, warnings_helper
from textwrap import dedent
+import functools
-from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
+from codeop import compile_command, CommandCompiler, Compile
+from codeop import PyCF_DONT_IMPLY_DEDENT, PyCF_ONLY_AST
+import ast
+
+
+WRAPPING_COMPILERS = [compile_command, CommandCompiler()]
+RAW_COMPILERS = [Compile()]
+COMPILERS = WRAPPING_COMPILERS + RAW_COMPILERS
-class CodeopTests(unittest.TestCase):
- def assertValid(self, str, symbol='single'):
+class CodeopTests(unittest.TestCase):
+ def assertValid(self, str, symbol='single', *, compiler):
'''succeed iff str is a valid piece of code'''
expected = compile(str, "", symbol, PyCF_DONT_IMPLY_DEDENT)
- self.assertEqual(compile_command(str, "", symbol), expected)
+ self.assertEqual(compiler(str, "", symbol), expected)
- def assertIncomplete(self, str, symbol='single'):
+ def assertIncomplete(self, str, symbol='single', *, compiler):
'''succeed iff str is the start of a valid piece of code'''
- self.assertEqual(compile_command(str, symbol=symbol), None)
-
- def assertInvalid(self, str, symbol='single', is_syntax=1):
+ if compiler in WRAPPING_COMPILERS:
+ self.assertEqual(compiler(str, "", symbol=symbol), None)
+ else:
+ # Compile has should raise like built-in compile
+ with self.assertRaises(SyntaxError) as cm_original_error:
+ compile(str, "", symbol, compiler.flags)
+ expected_error = cm_original_error.exception
+ with self.assertRaises(type(expected_error)) as cm_wrapped_error:
+ compiler(str, "", symbol=symbol)
+ self.assertEqual(
+ expected_error.args,
+ cm_wrapped_error.exception.args
+ )
+
+ def assertInvalid(self, str, symbol='single', is_syntax=1, *, compiler):
'''succeed iff str is the start of an invalid piece of code'''
try:
- compile_command(str,symbol=symbol)
+ compiler(str,"", symbol=symbol)
self.fail("No exception raised for invalid code")
except SyntaxError:
self.assertTrue(is_syntax)
except OverflowError:
self.assertTrue(not is_syntax)
- def test_valid(self):
- av = self.assertValid
-
- # special case
- self.assertEqual(compile_command(""),
- compile("pass", "", 'single',
- PyCF_DONT_IMPLY_DEDENT))
- self.assertEqual(compile_command("\n"),
- compile("pass", "", 'single',
- PyCF_DONT_IMPLY_DEDENT))
-
+ @subTests('compiler', WRAPPING_COMPILERS)
+ def test_empty(self, compiler):
+ self.assertEqual(
+ compiler("", "", 'single'),
+ compile("pass", "", 'single', PyCF_DONT_IMPLY_DEDENT))
+ self.assertEqual(
+ compiler("\n", "", 'single'),
+ compile("pass", "", 'single', PyCF_DONT_IMPLY_DEDENT))
+
+ @subTests('compiler', COMPILERS)
+ def test_valid(self, compiler):
+ av = functools.partial(self.assertValid, compiler=compiler)
av("a = 1")
av("\na = 1")
av("a = 1\n")
@@ -93,8 +114,9 @@ def test_valid(self):
av("@a.b.c\ndef f():\n pass\n")
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != None
- def test_incomplete(self):
- ai = self.assertIncomplete
+ @subTests('compiler', COMPILERS)
+ def test_incomplete(self, compiler):
+ ai = functools.partial(self.assertIncomplete, compiler=compiler)
ai("(a **")
ai("(a,b,")
@@ -227,8 +249,9 @@ def test_incomplete(self):
ai('a = f"""')
ai('a = \\')
- def test_invalid(self):
- ai = self.assertInvalid
+ @subTests('compiler', COMPILERS)
+ def test_invalid(self, compiler):
+ ai = functools.partial(self.assertInvalid, compiler=compiler)
ai("a b")
ai("a @")
@@ -264,8 +287,9 @@ def test_invalid(self):
ai("[i for i in range(10)] = (1, 2, 3)")
- def test_invalid_exec(self):
- ai = self.assertInvalid
+ @subTests('compiler', COMPILERS)
+ def test_invalid_exec(self, compiler):
+ ai = functools.partial(self.assertInvalid, compiler=compiler)
ai("raise = 4", symbol="exec")
ai('def a-b', symbol='exec')
ai('await?', symbol='exec')
@@ -273,59 +297,96 @@ def test_invalid_exec(self):
ai('a await raise b', symbol='exec')
ai('a await raise b?+1', symbol='exec')
- def test_filename(self):
- self.assertEqual(compile_command("a = 1\n", "abc").co_filename,
- compile("a = 1\n", "abc", 'single').co_filename)
- self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename,
- compile("a = 1\n", "def", 'single').co_filename)
-
- def test_warning(self):
+ @subTests('compiler', COMPILERS)
+ def test_filename(self, compiler):
+ self.assertEqual(
+ compiler("a = 1\n", "abc", "single").co_filename,
+ compile("a = 1\n", "abc", 'single').co_filename
+ )
+ self.assertNotEqual(
+ compiler("a = 1\n", "abc", "single").co_filename,
+ compile("a = 1\n", "def", 'single').co_filename
+ )
+
+ def assertReturnsModule(self, code, compiler):
+ retval = compiler(code, "", 'exec', PyCF_ONLY_AST)
+ self.assertIsInstance(retval, ast.Module)
+
+ @subTests('compiler', RAW_COMPILERS)
+ def test_ast_return_value(self, compiler):
+ validate_ast = self.assertReturnsModule
+ validate_ast("x = 5", compiler)
+ validate_ast("\nx = 5", compiler)
+ validate_ast("x = 5\n", compiler)
+ validate_ast("x = 5\n\n", compiler)
+ validate_ast("\n\nx = 5\n\n", compiler)
+
+ @subTests('compiler', COMPILERS)
+ def test_warning(self, compiler):
# Test that the warning is only returned once.
with warnings_helper.check_warnings(
('"is" with \'str\' literal', SyntaxWarning),
('"\\\\e" is an invalid escape sequence', SyntaxWarning),
) as w:
- compile_command(r"'\e' is 0")
- self.assertEqual(len(w.warnings), 2)
+ compiler(r"'\e' is 0", "", "single")
+ self.assertEqual(len(w.warnings), 2)
# bpo-41520: check SyntaxWarning treated as an SyntaxError
with warnings.catch_warnings(), self.assertRaises(SyntaxError):
warnings.simplefilter('error', SyntaxWarning)
- compile_command('1 is 1', symbol='exec')
+ compiler('1 is 1', "", 'exec')
# Check SyntaxWarning treated as an SyntaxError
with warnings.catch_warnings(), self.assertRaises(SyntaxError):
warnings.simplefilter('error', SyntaxWarning)
- compile_command(r"'\e'", symbol='exec')
+ compiler(r"'\e'", "", 'exec')
- def test_incomplete_warning(self):
+ @subTests('compiler', WRAPPING_COMPILERS)
+ def test_incomplete_warning(self, compiler):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
- self.assertIncomplete("'\\e' + (")
+ compiler("'\\e' + (")
self.assertEqual(w, [])
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1
- def test_invalid_warning(self):
+ @subTests('compiler', RAW_COMPILERS)
+ def test_raw_raises_error(self, compiler):
+ warnings_cm = warnings_helper.check_warnings(
+ ('"\\\\e" is an invalid esceape sequence', SyntaxWarning)
+ )
+ with self.assertRaises(SyntaxError), warnings_cm as w:
+ compiler("'\\e' + (", "", 'single')
+ self.assertEqual(len(w.warnings), 1)
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1
+ @subTests('compiler', COMPILERS)
+ def test_invalid_warning(self, compiler):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
- self.assertInvalid("'\\e' 1")
+ self.assertInvalid("'\\e' 1", compiler=compiler)
self.assertEqual(len(w), 1)
- self.assertEqual(w[0].category, SyntaxWarning)
- self.assertRegex(str(w[0].message), 'invalid escape sequence')
- self.assertEqual(w[0].filename, '')
-
- def assertSyntaxErrorMatches(self, code, message):
- with self.subTest(code):
- with self.assertRaisesRegex(SyntaxError, message):
- compile_command(code, symbol='exec')
-
- def test_syntax_errors(self):
- self.assertSyntaxErrorMatches(
- dedent("""\
+ for warning in w:
+ self.assertEqual(warning.category, SyntaxWarning)
+ self.assertRegex(str(warning), 'invalid escape sequence')
+ self.assertEqual(warning.filename, '')
+
+ @subTests('compiler', COMPILERS)
+ def test_syntax_errors(self, compiler):
+ code = dedent("""\
def foo(x,x):
pass
- """), "duplicate argument 'x' in function definition")
-
+ """)
+ message = "duplicate argument 'x' in function definition"
+ with self.assertRaisesRegex(SyntaxError, message):
+ compiler(code, "", 'exec')
+
+ @subTests('compiler', RAW_COMPILERS)
+ def test_future_imports(self, compiler):
+ original_flags = compiler.flags
+ compiler('from __future__ import annotations', "", 'single')
+ self.assertGreater(compiler.flags, original_flags)
+ # reset flags to ensure test has no side-effects
+ compiler.flags = original_flags
if __name__ == "__main__":
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index bb178d487e5..be5e7d7feee 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -550,6 +550,24 @@ def temporary_pycache_prefix(self):
finally:
sys.pycache_prefix = old_prefix
+ @contextlib.contextmanager
+ def no_pycache_prefix(self):
+ """Ignore any ambient pycache prefix for the duration of the test.
+
+ Some tests assume bytecode is written next to the source in a
+ __pycache__ directory. When the test suite is run with
+ PYTHONPYCACHEPREFIX set, neutralize it both in this process (used by
+ cache_from_source) and in any spawned subprocesses.
+ """
+ old_prefix = sys.pycache_prefix
+ sys.pycache_prefix = None
+ try:
+ with os_helper.EnvironmentVarGuard() as env:
+ env.unset('PYTHONPYCACHEPREFIX')
+ yield
+ finally:
+ sys.pycache_prefix = old_prefix
+
def _get_run_args(self, args):
return [*support.optim_args_from_interpreter_flags(),
'-S', '-m', 'compileall',
@@ -648,15 +666,16 @@ def test_legacy_paths(self):
def test_multiple_runs(self):
# Bug 8527 reported that multiple calls produced empty
# __pycache__/__pycache__ directories.
- self.assertRunOK('-q', self.pkgdir)
- # Verify the __pycache__ directory contents.
- self.assertTrue(os.path.exists(self.pkgdir_cachedir))
- cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
- self.assertFalse(os.path.exists(cachecachedir))
- # Call compileall again.
- self.assertRunOK('-q', self.pkgdir)
- self.assertTrue(os.path.exists(self.pkgdir_cachedir))
- self.assertFalse(os.path.exists(cachecachedir))
+ with self.no_pycache_prefix():
+ self.assertRunOK('-q', self.pkgdir)
+ # Verify the __pycache__ directory contents.
+ self.assertTrue(os.path.exists(self.pkgdir_cachedir))
+ cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
+ self.assertFalse(os.path.exists(cachecachedir))
+ # Call compileall again.
+ self.assertRunOK('-q', self.pkgdir)
+ self.assertTrue(os.path.exists(self.pkgdir_cachedir))
+ self.assertFalse(os.path.exists(cachecachedir))
@without_source_date_epoch # timestamp invalidation test
def test_force(self):
@@ -729,10 +748,13 @@ def test_symlink_loop(self):
script_helper.make_pkg(pkg)
os.symlink('.', os.path.join(pkg, 'evil'))
os.symlink('.', os.path.join(pkg, 'evil2'))
- self.assertRunOK('-q', self.pkgdir)
- self.assertCompiled(os.path.join(
- self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
- ))
+ # This relies on the __pycache__ layout (shared across the symlinked
+ # paths), so neutralize any ambient PYTHONPYCACHEPREFIX.
+ with self.no_pycache_prefix():
+ self.assertRunOK('-q', self.pkgdir)
+ self.assertCompiled(os.path.join(
+ self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
+ ))
def test_quiet(self):
noisy = self.assertRunOK(self.pkgdir)
@@ -819,13 +841,16 @@ def test_include_on_stdin(self):
f2 = script_helper.make_script(self.pkgdir, 'f2', '')
f3 = script_helper.make_script(self.pkgdir, 'f3', '')
f4 = script_helper.make_script(self.pkgdir, 'f4', '')
- p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
- p.stdin.write((f3+os.linesep).encode('ascii'))
- script_helper.kill_python(p)
- self.assertNotCompiled(f1)
- self.assertNotCompiled(f2)
- self.assertCompiled(f3)
- self.assertNotCompiled(f4)
+ # spawn_python() runs with -E, ignoring PYTHONPYCACHEPREFIX, so make
+ # cache_from_source() in this process agree by neutralizing it too.
+ with self.no_pycache_prefix():
+ p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
+ p.stdin.write((f3+os.linesep).encode('ascii'))
+ script_helper.kill_python(p)
+ self.assertNotCompiled(f1)
+ self.assertNotCompiled(f2)
+ self.assertCompiled(f3)
+ self.assertNotCompiled(f4)
def test_compiles_as_much_as_possible(self):
bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error')
diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py
index 96f42183296..962d75abc2d 100644
--- a/Lib/test/test_dataclasses/__init__.py
+++ b/Lib/test/test_dataclasses/__init__.py
@@ -2754,6 +2754,55 @@ def __eq__(self, other):
self.assertEqual(C(1), 5)
self.assertNotEqual(C(1), 1)
+ def test_eq_field_by_field(self):
+ @dataclasses.dataclass
+ class Point:
+ x: int
+ y: int
+
+ p1 = Point(1, 2)
+ p2 = Point(1, 2)
+ p3 = Point(2, 1)
+ self.assertEqual(p1, p2)
+ self.assertNotEqual(p1, p3)
+
+ def test_eq_type_check(self):
+ @dataclasses.dataclass
+ class A:
+ x: int
+
+ @dataclasses.dataclass
+ class B:
+ x: int
+
+ a = A(1)
+ b = B(1)
+ self.assertNotEqual(a, b)
+
+ def test_eq_custom_field(self):
+ class AlwaysEqual(int):
+ def __eq__(self, other):
+ return True
+
+ @dataclasses.dataclass
+ class Foo:
+ x: AlwaysEqual
+ y: int
+
+ f1 = Foo(AlwaysEqual(1), 2)
+ f2 = Foo(AlwaysEqual(2), 2)
+ self.assertEqual(f1, f2)
+
+ def test_eq_nan_field(self):
+ @dataclasses.dataclass
+ class D:
+ x: float
+
+ nan = float('nan')
+ d1 = D(nan)
+ d2 = D(nan)
+ self.assertNotEqual(d1, d2)
+
class TestOrdering(unittest.TestCase):
def test_functools_total_ordering(self):
@@ -3290,6 +3339,47 @@ def test_non_frozen_normal_derived(self):
class D:
x: int
y: int = 10
+ z: int = 1
+
+ @property
+ def readonly(self) -> int:
+ return self.x
+
+ @property
+ def prop(self) -> int:
+ return self.z
+
+ @prop.setter
+ def prop(self, val: int) -> None:
+ object.__setattr__(self, 'z', val)
+
+ @prop.deleter
+ def prop(self) -> None:
+ object.__setattr__(self, 'z', 0)
+
+ d = D(5)
+ self.assertEqual(d.x, 5)
+ self.assertEqual(d.y, 10)
+ self.assertEqual(d.z, 1)
+ self.assertEqual(d.readonly, 5)
+ self.assertEqual(d.prop, 1)
+
+ with self.assertRaises(FrozenInstanceError):
+ d.x = 5
+ with self.assertRaises(FrozenInstanceError):
+ d.readonly = 5
+ with self.assertRaises(FrozenInstanceError):
+ d.z = 5
+ with self.assertRaises(FrozenInstanceError):
+ d.prop = 5
+ with self.assertRaises(FrozenInstanceError):
+ del d.prop
+
+ self.assertEqual(d.x, 5)
+ self.assertEqual(d.y, 10)
+ self.assertEqual(d.z, 1)
+ self.assertEqual(d.readonly, 5)
+ self.assertEqual(d.prop, 1)
class S(D):
pass
@@ -3297,16 +3387,40 @@ class S(D):
s = S(3)
self.assertEqual(s.x, 3)
self.assertEqual(s.y, 10)
+ self.assertEqual(s.z, 1)
+ self.assertEqual(s.readonly, 3)
+ self.assertEqual(s.prop, 1)
+ # Can set new attrs:
s.cached = True
+ self.assertTrue(s.cached)
+ # Can mutate them:
+ s.cached = False
+ self.assertFalse(s.cached)
+
+ # Can also change writable properties:
+ with self.assertRaisesRegex(
+ AttributeError,
+ 'object has no setter',
+ ) as cm:
+ s.readonly = 5
+ self.assertNotIsInstance(cm.exception, FrozenInstanceError)
+ s.prop = 2
+ self.assertEqual(s.x, 3)
+ self.assertEqual(s.readonly, 3)
+ self.assertEqual(s.prop, 2)
+ self.assertEqual(s.z, 2)
# But can't change the frozen attributes.
with self.assertRaises(FrozenInstanceError):
s.x = 5
with self.assertRaises(FrozenInstanceError):
s.y = 5
+ with self.assertRaises(FrozenInstanceError):
+ s.z = 5
self.assertEqual(s.x, 3)
self.assertEqual(s.y, 10)
- self.assertEqual(s.cached, True)
+ self.assertEqual(s.z, 2)
+ self.assertIs(s.cached, False)
with self.assertRaises(FrozenInstanceError):
del s.x
@@ -3314,11 +3428,26 @@ class S(D):
with self.assertRaises(FrozenInstanceError):
del s.y
self.assertEqual(s.y, 10)
+ with self.assertRaisesRegex(
+ AttributeError,
+ 'object has no deleter',
+ ) as cm:
+ del s.readonly
+ self.assertNotIsInstance(cm.exception, FrozenInstanceError)
+ self.assertEqual(s.x, 3)
+ self.assertEqual(s.readonly, 3)
del s.cached
self.assertNotHasAttr(s, 'cached')
- with self.assertRaises(AttributeError) as cm:
+ with self.assertRaisesRegex(
+ AttributeError,
+ "object has no attribute 'cached'",
+ ) as cm:
del s.cached
self.assertNotIsInstance(cm.exception, FrozenInstanceError)
+ del s.prop
+ self.assertEqual(s.z, 0)
+ self.assertEqual(s.prop, 0)
+ del s.prop
def test_non_frozen_normal_derived_from_empty_frozen(self):
@dataclass(frozen=True)
diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index da919fae6e4..fc39240fb19 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -815,6 +815,15 @@ class X(C, int()):
class X(int(), C):
pass
+ @unittest.skipIf(_testcapi is None, 'need the _testcapi module')
+ def test_type_with_null_new_metaclass(self):
+ metaclass = _testcapi.HeapCTypeMetaclassNullNew
+ base = _testcapi.pytype_fromspec_meta(metaclass)
+
+ # Exercise type_new's metaclass selection path, not a direct call.
+ with self.assertRaisesRegex(TypeError, r"cannot create '.*' instances"):
+ type("Derived", (base,), {})
+
def test_module_subclasses(self):
# Testing Python subclass of module...
log = []
@@ -3698,6 +3707,7 @@ def f(a): return a
self.assertEqual(ba, b'abc\xbd?')
@unittest.skip("TODO: RUSTPYTHON; rustpython segmentation fault")
+ @support.skip_if_huge_c_stack()
@support.skip_wasi_stack_overflow()
@support.skip_emscripten_stack_overflow()
def test_recursive_call(self):
@@ -4914,6 +4924,7 @@ class Thing:
# CALL_METHOD_DESCRIPTOR_O
deque.append(thing, thing)
+ @support.skip_if_huge_c_stack()
@support.skip_emscripten_stack_overflow()
@support.skip_wasi_stack_overflow()
def test_repr_as_str(self):
diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py
index 4e5b9184025..024d20a63a1 100644
--- a/Lib/test/test_dictviews.py
+++ b/Lib/test/test_dictviews.py
@@ -2,7 +2,9 @@
import copy
import pickle
import unittest
-from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit
+from test.support import (skip_emscripten_stack_overflow,
+ skip_wasi_stack_overflow, run_with_limited_c_stack,
+ exceeds_recursion_limit)
class DictSetTest(unittest.TestCase):
@@ -278,6 +280,7 @@ def test_recursive_repr(self):
self.assertIsInstance(r, str)
@unittest.skip("TODO: RUSTPYTHON; segfault")
+ @run_with_limited_c_stack()
@skip_wasi_stack_overflow()
@skip_emscripten_stack_overflow()
def test_deeply_nested_repr(self):
diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py
index 5cb54cff9b7..c8b85fe8692 100644
--- a/Lib/test/test_enumerate.py
+++ b/Lib/test/test_enumerate.py
@@ -3,8 +3,11 @@
import sys
import pickle
import gc
+import threading
+
from test import support
+from test.support import threading_helper
class G:
'Sequence using __getitem__'
@@ -292,5 +295,28 @@ def enum(self, iterable, start=sys.maxsize + 1):
(sys.maxsize+3,'c')]
+@threading_helper.requires_working_threading()
+class TestThreadSafety(EnumerateStartTestCase):
+ def test_thread_safety_while_iterating(self):
+ # gh-153932: calling reduce while iterating should pass with TSAN
+
+ en = enumerate(range(10_000))
+ stop = threading.Event()
+
+ def advance():
+ for _ in en:
+ pass
+ stop.set()
+
+ def read():
+ while not stop.is_set():
+ en.__reduce__()
+
+ threads = [threading.Thread(target=advance), threading.Thread(target=read)]
+
+ with threading_helper.start_threads(threads):
+ pass
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py
index 7ad3aa8a527..43045a981de 100644
--- a/Lib/test/test_memoryio.py
+++ b/Lib/test/test_memoryio.py
@@ -869,7 +869,10 @@ def test_setstate(self):
@support.cpython_only
def test_sizeof(self):
- basesize = support.calcobjsize('P2n2Pn')
+ if support.Py_GIL_DISABLED:
+ basesize = support.calcobjsize('P2n2Pni')
+ else:
+ basesize = support.calcobjsize('P2n2Pn')
check = self.check_sizeof
self.assertEqual(object.__sizeof__(io.BytesIO()), basesize)
check(io.BytesIO(), basesize )
@@ -925,7 +928,6 @@ def test_cow_mutable(self):
def test_flags(self):
return super().test_flags()
-
class CStringIOTest(PyStringIOTest):
ioclass = io.StringIO
UnsupportedOperation = io.UnsupportedOperation
@@ -996,6 +998,7 @@ def __str__(self):
def test_flags(self):
return super().test_flags()
+
class CStringIOPickleTest(PyStringIOPickleTest):
UnsupportedOperation = io.UnsupportedOperation
@@ -1005,5 +1008,6 @@ def __new__(cls, *args, **kwargs):
def __init__(self, *args, **kwargs):
pass
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py
index ae7935ac07e..01678de56a9 100644
--- a/Lib/test/test_ordered_dict.py
+++ b/Lib/test/test_ordered_dict.py
@@ -878,6 +878,41 @@ def side_effect(self):
self.assertDictEqual(dict1, dict.fromkeys((0, 4.2)))
self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2)))
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised by copy
+ def test_issue148660_copy_clear_in_key_eq(self):
+ # gh-148660: od.copy() must not crash when a key's __eq__ clears od
+ # while copy() is inserting into the new dict.
+ armed = False
+ calls = 0
+ class Key:
+ def __hash__(self):
+ return 1
+ def __eq__(self, other):
+ nonlocal calls
+ if armed:
+ calls += 1
+ if calls == 2:
+ od.clear()
+ return self is other
+ od = self.OrderedDict()
+ od[Key()] = "v1"
+ od[Key()] = "v2"
+ armed = True
+ msg = "OrderedDict mutated during iteration"
+ self.assertRaisesRegex(RuntimeError, msg, od.copy)
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised by copy
+ def test_issue148660_copy_clear_in_subclass_getitem(self):
+ # gh-148660: od.copy() must not crash when a subclass __getitem__
+ # clears od.
+ class OD(self.OrderedDict):
+ def __getitem__(self, key):
+ od.clear()
+ return "v"
+ od = OD([(1, "v1"), (2, "v2")])
+ msg = "OrderedDict mutated during iteration"
+ self.assertRaisesRegex(RuntimeError, msg, od.copy)
+
@unittest.skipUnless(c_coll, 'requires the C version of the collections module')
class CPythonOrderedDictTests(OrderedDictTests,
diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py
index 0217ebd132b..c231126d58c 100644
--- a/Lib/test/test_random.py
+++ b/Lib/test/test_random.py
@@ -1074,6 +1074,21 @@ def test_avg_std(self):
self.assertAlmostEqual(s2/(N-1), sigmasqrd, places=2,
msg='%s%r' % (variate.__name__, args))
+ def test_binomialvariate_log_zero(self):
+ # gh-149222: Variety random() return 0.0 no input Error
+ with unittest.mock.patch.object(random.Random, 'random', side_effect=[0.0] + [0.5] * 20):
+ result = random.binomialvariate(10, 0.5)
+ self.assertIsInstance(result, int)
+ self.assertIn(result, range(11))
+
+ def test_binomialvariate_btrs_random_zero(self):
+ for p, expected in ((0.25, 25), (0.75, 75)):
+ with self.subTest(p=p):
+ g = random.Random()
+ with unittest.mock.patch.object(
+ g, 'random', side_effect=(0.0, 0.5, 0.5)):
+ self.assertEqual(g.binomialvariate(100, p), expected)
+
def test_constant(self):
g = random.Random()
N = 100
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index cd1477037e9..1ec64da064d 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -188,6 +188,8 @@ def test_request_rate(self):
parsed_request_rate.seconds,
self.request_rate.seconds
)
+ else:
+ self.assertIsNone(parsed_request_rate)
class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
@@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
bad = []
+class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase):
+ # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+ # them, so the directive must be silently ignored, not raise.
+ robots_txt = """\
+User-Agent: *
+Disallow: /tmp/
+Crawl-delay: ²
+Request-rate: ²/5
+ """
+ good = ['/foo.html']
+ bad = ['/tmp/']
+ crawl_delay = None
+ request_rate = None
+
+
+class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase):
+ robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: 5/²
+ """
+ good = ['/foo.html']
+ request_rate = None
+ bad = ['/tmp/']
+
+
class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
# also test that Allow and Diasallow works well with each other
robots_txt = """\
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 4280e8d450b..a1252fb6603 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -12,7 +12,9 @@
import sys
import unicodedata
import unittest
+import weakref
from test.support import (
+ gc_collect,
open_urlresource,
requires_resource,
script_helper,
@@ -30,14 +32,33 @@ def iterallchars():
maxunicode = 0xffff if quicktest else sys.maxunicode
return map(chr, range(maxunicode + 1))
+
+def check_version(testfile):
+ hdr = testfile.readline()
+ return unicodedata.unidata_version in hdr
+
+
+def download_test_data_file(filename):
+ TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{filename}"
+
+ try:
+ return open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version)
+ except PermissionError:
+ raise unittest.SkipTest(
+ f"Permission error when downloading {TESTDATAURL} "
+ f"into the test data directory"
+ )
+ except (OSError, HTTPException) as exc:
+ raise unittest.SkipTest(f"Failed to download {TESTDATAURL}: {exc}")
+
+
class UnicodeMethodsTest(unittest.TestCase):
# update this, if the database changes
- expectedchecksum = ('486bf97d506d0ccf0e463fd1f40c51029805af5a'
+ expectedchecksum = ('47a99fa654ef1f50e89d2e9697b7b041fccb5a05'
if quicktest else
- '9e43ee3929471739680c0e705482b4ae1c4122e4')
+ '8b2615a9fc627676cbc0b6fac0191177df97ef5f')
- @unittest.expectedFailure # TODO: RUSTPYTHON; + 9e43ee3929471739680c0e705482b4ae1c4122e4
def test_method_checksum(self):
h = hashlib.sha1()
for char in iterallchars():
@@ -84,17 +105,9 @@ def test_method_checksum(self):
self.assertEqual(result, self.expectedchecksum)
-class UnicodeFunctionsTest(unittest.TestCase):
- db = unicodedata
- old = False
-
- # Update this if the database changes. Make sure to do a full rebuild
- # (e.g. 'make distclean && make') to get the correct checksum.
- expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0'
- if quicktest else
- '46ca89d9fe34881d0be3a4a4b29f5aa8c019640c')
+class BaseUnicodeFunctionsTest:
- @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'digit'
+ @unittest.skip # TODO: RUSTPYTHON; AssertionError: '7bda75e48a961a01ab328358980cefc5c0a1666d' != '68cd01e2c680b851c1fcab012efb5635'
def test_function_checksum(self):
db = self.db
data = []
@@ -118,6 +131,7 @@ def test_function_checksum(self):
result = h.hexdigest()
self.assertEqual(result, self.expectedchecksum)
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
def test_name(self):
name = self.db.name
self.assertRaises(ValueError, name, '\0')
@@ -149,12 +163,16 @@ def test_name(self):
'EGYPTIAN HIEROGLYPH-13460')
self.assertEqual(name('\U000143FA', None), None if self.old else
'EGYPTIAN HIEROGLYPH-143FA')
+ self.assertEqual(name('\U00017000', None), None if self.old else
+ 'TANGUT IDEOGRAPH-17000')
self.assertEqual(name('\U00018B00', None), None if self.old else
'KHITAN SMALL SCRIPT CHARACTER-18B00')
self.assertEqual(name('\U00018CD5', None), None if self.old else
'KHITAN SMALL SCRIPT CHARACTER-18CD5')
self.assertEqual(name('\U00018CFF', None), None if self.old else
'KHITAN SMALL SCRIPT CHARACTER-18CFF')
+ self.assertEqual(name('\U00018D1E', None), None if self.old else
+ 'TANGUT IDEOGRAPH-18D1E')
self.assertEqual(name('\U0001B170', None), None if self.old else
'NUSHU CHARACTER-1B170')
self.assertEqual(name('\U0001B2FB', None), None if self.old else
@@ -164,8 +182,8 @@ def test_name(self):
'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE')
self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6')
self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D')
- self.assertEqual(name('\U000323AF', None), None if self.old else
- 'CJK UNIFIED IDEOGRAPH-323AF')
+ self.assertEqual(name('\U00033479', None), None if self.old else
+ 'CJK UNIFIED IDEOGRAPH-33479')
@requires_resource('cpu')
def test_name_inverse_lookup(self):
@@ -182,7 +200,7 @@ def test_no_names_in_pua(self):
char = chr(i)
self.assertRaises(ValueError, self.db.name, char)
- @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
def test_lookup_nonexistant(self):
# just make sure that lookup can fail
for nonexistent in [
@@ -225,7 +243,7 @@ def test_digit(self):
self.assertRaises(TypeError, self.db.digit, 'xx')
self.assertRaises(ValueError, self.db.digit, 'x')
- @unittest.skip # TODO: RUSTPYTHON; - None != 1e+20 (for 3.2.0; passes on latest)
+ @unittest.skip # TODO: RUSTPYTHON; None != 1e+20 (for 3.2.0; passes on latest)
def test_numeric(self):
self.assertEqual(self.db.numeric('A',None), None)
self.assertEqual(self.db.numeric('9'), 9)
@@ -306,7 +324,6 @@ def test_category(self):
self.assertRaises(TypeError, self.db.category)
self.assertRaises(TypeError, self.db.category, 'xx')
- # NOTE: RUSTPYTHON; This test is from 3.15. See RustPython#8548 for motivation.
def test_bidirectional(self):
self.assertEqual(self.db.bidirectional('\uFFFE'), '' if self.old else 'BN')
self.assertEqual(self.db.bidirectional(' '), 'WS')
@@ -336,7 +353,16 @@ def test_bidirectional(self):
self.assertRaises(TypeError, self.db.bidirectional)
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
- @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'D4CC 11B6' != '1111 1171 11B6'
+ def test_bidirectional_unassigned(self):
+ self.assertEqual(self.db.bidirectional('\u0378'), '' if self.old else 'L')
+ self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
+ self.assertEqual(self.db.bidirectional('\u20CF'), '' if self.old else 'ET')
+ self.assertEqual(self.db.bidirectional('\u0590'), '' if self.old else 'R')
+ self.assertEqual(self.db.bidirectional('\uFFFF'), '' if self.old else 'BN')
+ self.assertEqual(self.db.bidirectional('\U0001FFFE'), '' if self.old else 'BN')
+ self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ' 03A3 != ''
def test_decomposition(self):
self.assertEqual(self.db.decomposition('\uFFFE'),'')
self.assertEqual(self.db.decomposition('\u00bc'), ' 0031 2044 0034')
@@ -354,6 +380,8 @@ def test_decomposition(self):
self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else ' 04B1')
# New in 16.0.0
self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else ' 0041')
+ # New in 17.0.0
+ self.assertEqual(self.db.decomposition('\uA7F1'), '' if self.old else ' 0053')
# Hangul characters
self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161')
@@ -403,6 +431,8 @@ def test_combining(self):
self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220)
# New in 16.0.0
self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230)
+ # New in 17.0.0
+ self.assertEqual(self.db.combining('\u1ACF'), 0 if self.old else 230)
self.assertRaises(TypeError, self.db.combining)
self.assertRaises(TypeError, self.db.combining, 'xx')
@@ -589,6 +619,34 @@ def test_issue10254(self):
b = 'C\u0338' * 20 + '\xC7'
self.assertEqual(self.db.normalize('NFC', a), b)
+ def test_long_combining_mark_run(self):
+ # gh-149079: avoid quadratic canonical ordering.
+ payload = "a" + ("\u0300\u0327" * 32)
+ nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32)
+ nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31)
+
+ self.assertEqual(self.db.normalize("NFD", payload), nfd)
+ self.assertEqual(self.db.normalize("NFKD", payload), nfd)
+ self.assertEqual(self.db.normalize("NFC", payload), nfc)
+ self.assertEqual(self.db.normalize("NFKC", payload), nfc)
+
+ def test_combining_mark_run_fast_paths(self):
+ # gh-149079: cover short runs and already-sorted long runs.
+ short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300"
+ short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10)
+ short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9)
+ long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30)
+ long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29)
+
+ self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd)
+ self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd)
+ self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc)
+ self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc)
+ self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted)
+ self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted)
+ self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc)
+ self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc)
+
def test_issue29456(self):
# Fix #29456
u1176_str_a = '\u1100\u1176\u11a8'
@@ -641,6 +699,8 @@ def test_east_asian_width(self):
# New in 16.0.0
self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W')
self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W')
+ # New in 17.0.0
+ self.assertEqual(eaw('\U00016FF2'), 'N' if self.old else 'W')
@unittest.skip # TODO: RUSTPYTHON; AssertionError: 'N' != 'W' (passed on latest, fails on 3.2)
def test_east_asian_width_unassigned(self):
@@ -661,30 +721,434 @@ def test_east_asian_width_unassigned(self):
self.assertEqual(eaw(char), 'A')
self.assertIs(self.db.name(char, None), None)
+class UnicodeFunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
+ db = unicodedata
+ old = False
+
+ # Update this if the database changes. Make sure to do a full rebuild
+ # (e.g. 'make distclean && make') to get the correct checksum.
+ expectedchecksum = ('00b13fa975a60b1d3f490f1fc8c126ab24990c75'
+ if quicktest else
+ 'ebfc9dd281c2226998fd435744dd2e9321899beb')
+
+ @requires_resource('network')
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
+ def test_all_names(self):
+ TESTDATAFILE = "DerivedName.txt"
+ testdata = download_test_data_file(TESTDATAFILE)
+
+ with testdata:
+ self.run_name_tests(testdata)
+
+ def run_name_tests(self, testdata):
+ names_ref = {}
+
+ def parse_cp(s):
+ return int(s, 16)
+
+ # Parse data
+ for line in testdata:
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ raw_cp, name = line.split("; ")
+ # Check for a range
+ if ".." in raw_cp:
+ cp1, cp2 = map(parse_cp, raw_cp.split(".."))
+ # remove ‘*’ at the end
+ assert name[-1] == '*', (raw_cp, name)
+ name = name[:-1]
+ for cp in range(cp1, cp2 + 1):
+ names_ref[cp] = f"{name}{cp:04X}"
+ elif name[-1] == '*':
+ cp = parse_cp(raw_cp)
+ name = name[:-1]
+ names_ref[cp] = f"{name}{cp:04X}"
+ else:
+ assert '*' not in name, (raw_cp, name)
+ cp = parse_cp(raw_cp)
+ names_ref[cp] = name
+
+ for cp in range(0, sys.maxunicode + 1):
+ self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
+ def test_isxidstart(self):
+ self.assertTrue(self.db.isxidstart('S'))
+ self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM
+ self.assertTrue(self.db.isxidstart('\u0EC6')) # LAO KO LA
+ self.assertTrue(self.db.isxidstart('\u17DC')) # KHMER SIGN AVAKRAHASANYA
+ self.assertTrue(self.db.isxidstart('\uA015')) # YI SYLLABLE WU
+ self.assertTrue(self.db.isxidstart('\uFE7B')) # ARABIC KASRA MEDIAL FORM
+
+ self.assertFalse(self.db.isxidstart(' '))
+ self.assertFalse(self.db.isxidstart('0'))
+ self.assertRaises(TypeError, self.db.isxidstart)
+ self.assertRaises(TypeError, self.db.isxidstart, 'xx')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
+ def test_isxidcontinue(self):
+ self.assertTrue(self.db.isxidcontinue('S'))
+ self.assertTrue(self.db.isxidcontinue('_'))
+ self.assertTrue(self.db.isxidcontinue('0'))
+ self.assertTrue(self.db.isxidcontinue('\u00BA')) # MASCULINE ORDINAL INDICATOR
+ self.assertTrue(self.db.isxidcontinue('\u0640')) # ARABIC TATWEEL
+ self.assertTrue(self.db.isxidcontinue('\u0710')) # SYRIAC LETTER ALAPH
+ self.assertTrue(self.db.isxidcontinue('\u0B3E')) # ORIYA VOWEL SIGN AA
+ self.assertTrue(self.db.isxidcontinue('\u17D7')) # KHMER SIGN LEK TOO
+
+ self.assertFalse(self.db.isxidcontinue(' '))
+ self.assertRaises(TypeError, self.db.isxidcontinue)
+ self.assertRaises(TypeError, self.db.isxidcontinue, 'xx')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'grapheme_cluster_break'
+ def test_grapheme_cluster_break(self):
+ gcb = self.db.grapheme_cluster_break
+ self.assertEqual(gcb(' '), 'Other')
+ self.assertEqual(gcb('x'), 'Other')
+ self.assertEqual(gcb('\U0010FFFF'), 'Other')
+ self.assertEqual(gcb('\r'), 'CR')
+ self.assertEqual(gcb('\n'), 'LF')
+ self.assertEqual(gcb('\0'), 'Control')
+ self.assertEqual(gcb('\t'), 'Control')
+ self.assertEqual(gcb('\x1F'), 'Control')
+ self.assertEqual(gcb('\x7F'), 'Control')
+ self.assertEqual(gcb('\x9F'), 'Control')
+ self.assertEqual(gcb('\U000E0001'), 'Control')
+ self.assertEqual(gcb('\u0300'), 'Extend')
+ self.assertEqual(gcb('\u200C'), 'Extend')
+ self.assertEqual(gcb('\U000E01EF'), 'Extend')
+ self.assertEqual(gcb('\u1159'), 'L')
+ self.assertEqual(gcb('\u11F9'), 'T')
+ self.assertEqual(gcb('\uD788'), 'LV')
+ self.assertEqual(gcb('\uD7A3'), 'LVT')
+ # New in 5.0.0
+ self.assertEqual(gcb('\u05BA'), 'Extend')
+ self.assertEqual(gcb('\u20EF'), 'Extend')
+ # New in 5.1.0
+ self.assertEqual(gcb('\u2064'), 'Control')
+ self.assertEqual(gcb('\uAA4D'), 'SpacingMark')
+ # New in 5.2.0
+ self.assertEqual(gcb('\u0816'), 'Extend')
+ self.assertEqual(gcb('\uA97C'), 'L')
+ self.assertEqual(gcb('\uD7C6'), 'V')
+ self.assertEqual(gcb('\uD7FB'), 'T')
+ # New in 6.0.0
+ self.assertEqual(gcb('\u093A'), 'Extend')
+ self.assertEqual(gcb('\U00011002'), 'SpacingMark')
+ # New in 6.1.0
+ self.assertEqual(gcb('\U000E0FFF'), 'Control')
+ self.assertEqual(gcb('\U00016F7E'), 'SpacingMark')
+ # New in 6.2.0
+ self.assertEqual(gcb('\U0001F1E6'), 'Regional_Indicator')
+ self.assertEqual(gcb('\U0001F1FF'), 'Regional_Indicator')
+ # New in 6.3.0
+ self.assertEqual(gcb('\u180E'), 'Control')
+ self.assertEqual(gcb('\u1A1B'), 'Extend')
+ # New in 7.0.0
+ self.assertEqual(gcb('\u0E33'), 'SpacingMark')
+ self.assertEqual(gcb('\u0EB3'), 'SpacingMark')
+ self.assertEqual(gcb('\U0001BCA3'), 'Control')
+ self.assertEqual(gcb('\U0001E8D6'), 'Extend')
+ self.assertEqual(gcb('\U0001163E'), 'SpacingMark')
+ # New in 8.0.0
+ self.assertEqual(gcb('\u08E3'), 'Extend')
+ self.assertEqual(gcb('\U00011726'), 'SpacingMark')
+ # New in 9.0.0
+ self.assertEqual(gcb('\u0600'), 'Prepend')
+ self.assertEqual(gcb('\U000E007F'), 'Extend')
+ self.assertEqual(gcb('\U00011CB4'), 'SpacingMark')
+ self.assertEqual(gcb('\u200D'), 'ZWJ')
+ # New in 10.0.0
+ self.assertEqual(gcb('\U00011D46'), 'Prepend')
+ self.assertEqual(gcb('\U00011D47'), 'Extend')
+ self.assertEqual(gcb('\U00011A97'), 'SpacingMark')
+ # New in 11.0.0
+ self.assertEqual(gcb('\U000110CD'), 'Prepend')
+ self.assertEqual(gcb('\u07FD'), 'Extend')
+ self.assertEqual(gcb('\U00011EF6'), 'SpacingMark')
+ # New in 12.0.0
+ self.assertEqual(gcb('\U00011A84'), 'Prepend')
+ self.assertEqual(gcb('\U00013438'), 'Control')
+ self.assertEqual(gcb('\U0001E2EF'), 'Extend')
+ self.assertEqual(gcb('\U00016F87'), 'SpacingMark')
+ # New in 13.0.0
+ self.assertEqual(gcb('\U00011941'), 'Prepend')
+ self.assertEqual(gcb('\U00016FE4'), 'Extend')
+ self.assertEqual(gcb('\U00011942'), 'SpacingMark')
+ # New in 14.0.0
+ self.assertEqual(gcb('\u0891'), 'Prepend')
+ self.assertEqual(gcb('\U0001E2AE'), 'Extend')
+ # New in 15.0.0
+ self.assertEqual(gcb('\U00011F02'), 'Prepend')
+ self.assertEqual(gcb('\U0001343F'), 'Control')
+ self.assertEqual(gcb('\U0001E4EF'), 'Extend')
+ self.assertEqual(gcb('\U00011F3F'), 'SpacingMark')
+ # New in 16.0.0
+ self.assertEqual(gcb('\U000113D1'), 'Prepend')
+ self.assertEqual(gcb('\U0001E5EF'), 'Extend')
+ self.assertEqual(gcb('\U0001612C'), 'SpacingMark')
+ self.assertEqual(gcb('\U00016D63'), 'V')
+ # New in 17.0.0
+ self.assertEqual(gcb('\u1AEB'), 'Extend')
+ self.assertEqual(gcb('\U00011B67'), 'SpacingMark')
+
+ self.assertRaises(TypeError, gcb)
+ self.assertRaises(TypeError, gcb, b'x')
+ self.assertRaises(TypeError, gcb, 120)
+ self.assertRaises(TypeError, gcb, '')
+ self.assertRaises(TypeError, gcb, 'xx')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'indic_conjunct_break'
+ def test_indic_conjunct_break(self):
+ incb = self.db.indic_conjunct_break
+ self.assertEqual(incb(' '), 'None')
+ self.assertEqual(incb('x'), 'None')
+ self.assertEqual(incb('\U0010FFFF'), 'None')
+ # New in 15.1.0
+ self.assertEqual(incb('\u094D'), 'Linker')
+ self.assertEqual(incb('\u0D4D'), 'Linker')
+ self.assertEqual(incb('\u0915'), 'Consonant')
+ self.assertEqual(incb('\u0D3A'), 'Consonant')
+ self.assertEqual(incb('\u0300'), 'Extend')
+ self.assertEqual(incb('\U0001E94A'), 'Extend')
+ # New in 16.0.0
+ self.assertEqual(incb('\u034F'), 'Extend')
+ self.assertEqual(incb('\U000E01EF'), 'Extend')
+ # New in 17.0.0
+ self.assertEqual(incb('\u1039'), 'Linker')
+ self.assertEqual(incb('\U00011F42'), 'Linker')
+ self.assertEqual(incb('\u1000'), 'Consonant')
+ self.assertEqual(incb('\U00011F33'), 'Consonant')
+ self.assertEqual(incb('\U0001E6F5'), 'Extend')
+
+ self.assertRaises(TypeError, incb)
+ self.assertRaises(TypeError, incb, b'x')
+ self.assertRaises(TypeError, incb, 120)
+ self.assertRaises(TypeError, incb, '')
+ self.assertRaises(TypeError, incb, 'xx')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'extended_pictographic'
+ def test_extended_pictographic(self):
+ ext_pict = self.db.extended_pictographic
+ self.assertIs(ext_pict(' '), False)
+ self.assertIs(ext_pict('x'), False)
+ self.assertIs(ext_pict('\U0010FFFF'), False)
+ # New in 13.0.0
+ self.assertIs(ext_pict('\xA9'), True)
+ self.assertIs(ext_pict('\u203C'), True)
+ self.assertIs(ext_pict('\U0001FAD6'), True)
+ self.assertIs(ext_pict('\U0001FFFD'), True)
+ # New in 17.0.0
+ self.assertIs(ext_pict('\u2388'), False)
+ self.assertIs(ext_pict('\U0001FA6D'), False)
+
+ self.assertRaises(TypeError, ext_pict)
+ self.assertRaises(TypeError, ext_pict, b'x')
+ self.assertRaises(TypeError, ext_pict, 120)
+ self.assertRaises(TypeError, ext_pict, '')
+ self.assertRaises(TypeError, ext_pict, 'xx')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
+ def test_grapheme_break(self):
+ def graphemes(*args):
+ return list(map(str, self.db.iter_graphemes(*args)))
+
+ self.assertRaises(TypeError, self.db.iter_graphemes)
+ self.assertRaises(TypeError, self.db.iter_graphemes, b'x')
+ self.assertRaises(TypeError, self.db.iter_graphemes, 'x', 0, 0, 0)
+
+ self.assertEqual(graphemes(''), [])
+ self.assertEqual(graphemes('abcd'), ['a', 'b', 'c', 'd'])
+ self.assertEqual(graphemes('abcd', 1), ['b', 'c', 'd'])
+ self.assertEqual(graphemes('abcd', 1, 3), ['b', 'c'])
+ self.assertEqual(graphemes('abcd', -3), ['b', 'c', 'd'])
+ self.assertEqual(graphemes('abcd', 1, -1), ['b', 'c'])
+ self.assertEqual(graphemes('abcd', 3, 1), [])
+ self.assertEqual(graphemes('abcd', 5), [])
+ self.assertEqual(graphemes('abcd', 0, 5), ['a', 'b', 'c', 'd'])
+ self.assertEqual(graphemes('abcd', -5), ['a', 'b', 'c', 'd'])
+ self.assertEqual(graphemes('abcd', 0, -5), [])
+ # GB3
+ self.assertEqual(graphemes('\r\n'), ['\r\n'])
+ # GB4
+ self.assertEqual(graphemes('\r\u0308'), ['\r', '\u0308'])
+ self.assertEqual(graphemes('\n\u0308'), ['\n', '\u0308'])
+ self.assertEqual(graphemes('\0\u0308'), ['\0', '\u0308'])
+ # GB5
+ self.assertEqual(graphemes('\u06dd\r'), ['\u06dd', '\r'])
+ self.assertEqual(graphemes('\u06dd\n'), ['\u06dd', '\n'])
+ self.assertEqual(graphemes('\u06dd\0'), ['\u06dd', '\0'])
+ # GB6
+ self.assertEqual(graphemes('\u1100\u1160'), ['\u1100\u1160'])
+ self.assertEqual(graphemes('\u1100\uAC00'), ['\u1100\uAC00'])
+ self.assertEqual(graphemes('\u1100\uAC01'), ['\u1100\uAC01'])
+ # GB7
+ self.assertEqual(graphemes('\uAC00\u1160'), ['\uAC00\u1160'])
+ self.assertEqual(graphemes('\uAC00\u11A8'), ['\uAC00\u11A8'])
+ self.assertEqual(graphemes('\u1160\u1160'), ['\u1160\u1160'])
+ self.assertEqual(graphemes('\u1160\u11A8'), ['\u1160\u11A8'])
+ # GB8
+ self.assertEqual(graphemes('\uAC01\u11A8'), ['\uAC01\u11A8'])
+ self.assertEqual(graphemes('\u11A8\u11A8'), ['\u11A8\u11A8'])
+ # GB9
+ self.assertEqual(graphemes('a\u0300'), ['a\u0300'])
+ self.assertEqual(graphemes('a\u200D'), ['a\u200D'])
+ # GB9a
+ self.assertEqual(graphemes('\u0905\u0903'), ['\u0905\u0903'])
+ # GB9b
+ self.assertEqual(graphemes('\u06dd\u0661'), ['\u06dd\u0661'])
+ # GB9c
+ self.assertEqual(graphemes('\u0915\u094d\u0924'),
+ ['\u0915\u094d\u0924'])
+ self.assertEqual(graphemes('\u0915\u094D\u094D\u0924'),
+ ['\u0915\u094D\u094D\u0924'])
+ self.assertEqual(graphemes('\u0915\u094D\u0924\u094D\u092F'),
+ ['\u0915\u094D\u0924\u094D\u092F'])
+ # GB11
+ self.assertEqual(graphemes(
+ '\U0001F9D1\U0001F3FE\u200D\u2764\uFE0F'
+ '\u200D\U0001F48B\u200D\U0001F9D1\U0001F3FC'),
+ ['\U0001F9D1\U0001F3FE\u200D\u2764\uFE0F'
+ '\u200D\U0001F48B\u200D\U0001F9D1\U0001F3FC'])
+ # GB12
+ self.assertEqual(graphemes(
+ '\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
+ ['\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])
+ # GB13
+ self.assertEqual(graphemes(
+ 'a\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
+ ['a', '\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
+ def test_block(self):
+ self.assertEqual(self.db.block('\u0000'), 'Basic Latin')
+ self.assertEqual(self.db.block('\u0041'), 'Basic Latin')
+ self.assertEqual(self.db.block('\u007F'), 'Basic Latin')
+ self.assertEqual(self.db.block('\u0080'), 'Latin-1 Supplement')
+ self.assertEqual(self.db.block('\u00FF'), 'Latin-1 Supplement')
+ self.assertEqual(self.db.block('\u1159'), 'Hangul Jamo')
+ self.assertEqual(self.db.block('\u11F9'), 'Hangul Jamo')
+ self.assertEqual(self.db.block('\uD788'), 'Hangul Syllables')
+ self.assertEqual(self.db.block('\uD7A3'), 'Hangul Syllables')
+ # New in 5.0.0
+ self.assertEqual(self.db.block('\u05BA'), 'Hebrew')
+ self.assertEqual(self.db.block('\u20EF'), 'Combining Diacritical Marks for Symbols')
+ # New in 5.1.0
+ self.assertEqual(self.db.block('\u2064'), 'General Punctuation')
+ self.assertEqual(self.db.block('\uAA4D'), 'Cham')
+ # New in 5.2.0
+ self.assertEqual(self.db.block('\u0816'), 'Samaritan')
+ self.assertEqual(self.db.block('\uA97C'), 'Hangul Jamo Extended-A')
+ self.assertEqual(self.db.block('\uD7C6'), 'Hangul Jamo Extended-B')
+ self.assertEqual(self.db.block('\uD7FB'), 'Hangul Jamo Extended-B')
+ # New in 6.0.0
+ self.assertEqual(self.db.block('\u093A'), 'Devanagari')
+ self.assertEqual(self.db.block('\U00011002'), 'Brahmi')
+ # New in 6.1.0
+ self.assertEqual(self.db.block('\U000E0FFF'), 'No_Block')
+ self.assertEqual(self.db.block('\U00016F7E'), 'Miao')
+ # New in 6.2.0
+ self.assertEqual(self.db.block('\U0001F1E6'), 'Enclosed Alphanumeric Supplement')
+ self.assertEqual(self.db.block('\U0001F1FF'), 'Enclosed Alphanumeric Supplement')
+ # New in 6.3.0
+ self.assertEqual(self.db.block('\u180E'), 'Mongolian')
+ self.assertEqual(self.db.block('\u1A1B'), 'Buginese')
+ # New in 7.0.0
+ self.assertEqual(self.db.block('\u0E33'), 'Thai')
+ self.assertEqual(self.db.block('\u0EB3'), 'Lao')
+ self.assertEqual(self.db.block('\U0001BCA3'), 'Shorthand Format Controls')
+ self.assertEqual(self.db.block('\U0001E8D6'), 'Mende Kikakui')
+ self.assertEqual(self.db.block('\U0001163E'), 'Modi')
+ # New in 8.0.0
+ self.assertEqual(self.db.block('\u08E3'), 'Arabic Extended-A')
+ self.assertEqual(self.db.block('\U00011726'), 'Ahom')
+ # New in 9.0.0
+ self.assertEqual(self.db.block('\u0600'), 'Arabic')
+ self.assertEqual(self.db.block('\U000E007F'), 'Tags')
+ self.assertEqual(self.db.block('\U00011CB4'), 'Marchen')
+ self.assertEqual(self.db.block('\u200D'), 'General Punctuation')
+ # New in 10.0.0
+ self.assertEqual(self.db.block('\U00011D46'), 'Masaram Gondi')
+ self.assertEqual(self.db.block('\U00011D47'), 'Masaram Gondi')
+ self.assertEqual(self.db.block('\U00011A97'), 'Soyombo')
+ # New in 11.0.0
+ self.assertEqual(self.db.block('\U000110CD'), 'Kaithi')
+ self.assertEqual(self.db.block('\u07FD'), 'NKo')
+ self.assertEqual(self.db.block('\U00011EF6'), 'Makasar')
+ # New in 12.0.0
+ self.assertEqual(self.db.block('\U00011A84'), 'Soyombo')
+ self.assertEqual(self.db.block('\U00013438'), 'Egyptian Hieroglyph Format Controls')
+ self.assertEqual(self.db.block('\U0001E2EF'), 'Wancho')
+ self.assertEqual(self.db.block('\U00016F87'), 'Miao')
+ # New in 13.0.0
+ self.assertEqual(self.db.block('\U00011941'), 'Dives Akuru')
+ self.assertEqual(self.db.block('\U00016FE4'), 'Ideographic Symbols and Punctuation')
+ self.assertEqual(self.db.block('\U00011942'), 'Dives Akuru')
+ # New in 14.0.0
+ self.assertEqual(self.db.block('\u0891'), 'Arabic Extended-B')
+ self.assertEqual(self.db.block('\U0001E2AE'), 'Toto')
+ # New in 15.0.0
+ self.assertEqual(self.db.block('\U00011F02'), 'Kawi')
+ self.assertEqual(self.db.block('\U0001343F'), 'Egyptian Hieroglyph Format Controls')
+ self.assertEqual(self.db.block('\U0001E4EF'), 'Nag Mundari')
+ self.assertEqual(self.db.block('\U00011F3F'), 'Kawi')
+ # New in 16.0.0
+ self.assertEqual(self.db.block('\U000113D1'), 'Tulu-Tigalari')
+ self.assertEqual(self.db.block('\U0001E5EF'), 'Ol Onal')
+ self.assertEqual(self.db.block('\U0001612C'), 'Gurung Khema')
+ self.assertEqual(self.db.block('\U00016D63'), 'Kirat Rai')
+ # New in 17.0.0
+ self.assertEqual(self.db.block('\u1AEB'), 'Combining Diacritical Marks Extended')
+ self.assertEqual(self.db.block('\U00011B67'), 'Sharada Supplement')
+ # Unassigned
+ self.assertEqual(self.db.block('\U00100000'), 'Supplementary Private Use Area-B')
+ self.assertEqual(self.db.block('\U0010FFFF'), 'Supplementary Private Use Area-B')
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
+ def test_block_invalid_input(self):
+ self.assertRaises(TypeError, self.db.block)
+ self.assertRaises(TypeError, self.db.block, b'x')
+ self.assertRaises(TypeError, self.db.block, 120)
+ self.assertRaises(TypeError, self.db.block, '')
+ self.assertRaises(TypeError, self.db.block, 'xx')
+
@unittest.expectedFailure # TODO: RUSTPYTHON; + N
def test_east_asian_width_9_0_changes(self):
return super().test_east_asian_width_9_0_changes()
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'D4CC 11B6' != '1111 1171 11B6'
+ def test_decomposition(self):
+ return super().test_decomposition()
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
+ def test_lookup_nonexistant(self):
+ return super().test_lookup_nonexistant()
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'digit'
+ def test_function_checksum(self):
+ return super().test_function_checksum()
+
-class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
+class Unicode_3_2_0_FunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
db = unicodedata.ucd_3_2_0
old = True
expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47'
if quicktest else
- 'caf1a7f2f380f927461837f1901ef20683f98683')
+ '68cd01e2c680b851c1fcab012efb5635b2229c2b')
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_normalization(self):
return super().test_normalization()
- @unittest.expectedSuccess # TODO: RUSTPYTHON
- def test_combining(self):
- return super().test_combining()
-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'LATIN SMALL LETTER D WITH CURL' != None
def test_name(self):
return super().test_name()
+ @unittest.expectedSuccess # TODO: RUSTPYTHON
+ def test_combining(self):
+ return super().test_combining()
+
class UnicodeMiscTest(unittest.TestCase):
db = unicodedata
@@ -712,6 +1176,23 @@ def test_failed_import_during_compiling(self):
"(can't load unicodedata module)"
self.assertIn(error, result.err.decode("ascii"))
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError Process return code is 1
+ def test_unicodedata_unload_reload(self):
+ # gh-149449: dropping unicodedata and running gc must not leave the
+ # cached _ucnhash_CAPI pointer dangling.
+ code = (
+ "import gc, sys\n"
+ "assert '\\N{GRINNING FACE}'.encode("
+ " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n"
+ "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '', 'exec')\n"
+ "del sys.modules['unicodedata']\n"
+ "gc.collect()\n"
+ "assert '\\N{WINKING FACE}'.encode("
+ " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n"
+ "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '', 'exec')\n"
+ )
+ script_helper.assert_python_ok("-c", code)
+
def test_decimal_numeric_consistent(self):
# Test that decimal and numeric are consistent,
# i.e. if a character has a decimal value,
@@ -788,13 +1269,20 @@ def test_linebreak_7643(self):
self.assertEqual(len(lines), 1,
r"%a should not be a linebreak" % c)
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
+ def test_segment_object(self):
+ segments = list(unicodedata.iter_graphemes('spa\u0300m'))
+ self.assertEqual(len(segments), 4, segments)
+ segment = segments[2]
+ self.assertEqual(segment.start, 2)
+ self.assertEqual(segment.end, 4)
+ self.assertEqual(str(segment), 'a\u0300')
+ self.assertEqual(repr(segment), '')
+ self.assertRaises(TypeError, iter, segment)
+ self.assertRaises(TypeError, len, segment)
-class NormalizationTest(unittest.TestCase):
- @staticmethod
- def check_version(testfile):
- hdr = testfile.readline()
- return unicodedata.unidata_version in hdr
+class NormalizationTest(unittest.TestCase):
@staticmethod
def unistr(data):
data = [int(x, 16) for x in data.split(" ")]
@@ -804,17 +1292,7 @@ def unistr(data):
@requires_resource('cpu')
def test_normalization(self):
TESTDATAFILE = "NormalizationTest.txt"
- TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"
-
- # Hit the exception early
- try:
- testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
- check=self.check_version)
- except PermissionError:
- self.skipTest(f"Permission error when downloading {TESTDATAURL} "
- f"into the test data directory")
- except (OSError, HTTPException) as exc:
- self.skipTest(f"Failed to download {TESTDATAURL}: {exc}")
+ testdata = download_test_data_file(TESTDATAFILE)
with testdata:
self.run_normalization_tests(testdata, unicodedata)
@@ -911,5 +1389,70 @@ class MyStr(str):
self.assertIs(type(normalize(form, MyStr(input_str))), str)
+class GraphemeBreakTest(unittest.TestCase):
+ @requires_resource('network')
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError module 'unicodedata' has no attribute 'iter_graphemes'
+ def test_grapheme_break(self):
+ TESTDATAFILE = "GraphemeBreakTest.txt"
+ testdata = download_test_data_file(TESTDATAFILE)
+
+ with testdata:
+ self.run_grapheme_break_tests(testdata)
+
+ def run_grapheme_break_tests(self, testdata):
+ for line in testdata:
+ line, _, comment = line.partition('#')
+ line = line.strip()
+ if not line:
+ continue
+ comment = comment.strip()
+
+ chunks = []
+ breaks = []
+ pos = 0
+ for field in line.replace('×', ' ').split():
+ if field == '÷':
+ chunks.append('')
+ breaks.append(pos)
+ else:
+ chunks[-1] += chr(int(field, 16))
+ pos += 1
+ self.assertEqual(chunks.pop(), '', line)
+ input = ''.join(chunks)
+ with self.subTest(line):
+ result = list(unicodedata.iter_graphemes(input))
+ self.assertEqual(list(map(str, result)), chunks, comment)
+ self.assertEqual([x.start for x in result], breaks[:-1], comment)
+ self.assertEqual([x.end for x in result], breaks[1:], comment)
+ for i in range(1, len(breaks) - 1):
+ result = list(unicodedata.iter_graphemes(input, breaks[i]))
+ self.assertEqual(list(map(str, result)), chunks[i:], comment)
+ self.assertEqual([x.start for x in result], breaks[i:-1], comment)
+ self.assertEqual([x.end for x in result], breaks[i+1:], comment)
+
+ @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
+ def test_reference_loops(self):
+ # Test that reference loops involving GraphemeBreakIterator or
+ # Segment can be broken by the garbage collector.
+ class S(str):
+ pass
+
+ s = S('abc')
+ s.ref = unicodedata.iter_graphemes(s)
+ wr = weakref.ref(s)
+ del s
+ self.assertIsNotNone(wr())
+ gc_collect()
+ self.assertIsNone(wr())
+
+ s = S('abc')
+ s.ref = next(unicodedata.iter_graphemes(s))
+ wr = weakref.ref(s)
+ del s
+ self.assertIsNotNone(wr())
+ gc_collect()
+ self.assertIsNone(wr())
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 2dd739b77b8..1e5f79998e7 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -467,6 +467,25 @@ def test_redirect_limit_independent(self):
finally:
self.unfakehttp()
+ def test_http_error_attribute_values(self):
+ hdrs = {
+ "Authorization": "Bearer foobar",
+ "Accept": "application/json"
+ }
+ err = urllib.error.HTTPError("http://something", 404, "foo", hdrs, None)
+ self.assertEqual(err.filename, "http://something")
+ self.assertEqual(err.code, 404)
+ self.assertEqual(err.msg, "foo")
+ self.assertEqual(err.reason, "foo")
+ self.assertEqual(err.hdrs, hdrs)
+ self.assertEqual(err.headers, hdrs)
+ err.close()
+
+ def test_http_error_default_fp(self):
+ err = urllib.error.HTTPError("http://something", 404, "foo", {}, None)
+ self.assertIsInstance(err.fp, io.BytesIO)
+ err.close()
+
def test_empty_socket(self):
# urlopen() raises OSError if the underlying socket does not send any
# data. (#1680230)
@@ -513,6 +532,11 @@ def test_ftp_nonexisting(self):
self.assertFalse(e.exception.filename)
self.assertTrue(e.exception.reason)
+ def test_url_error_stringified(self):
+ reason = 'sixseven'
+ err = urllib.error.URLError(reason)
+ self.assertEqual(str(err), f'')
+
class urlopen_DataTests(unittest.TestCase):
"""Test urlopen() opening a data URL."""
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 7d7f2fa00d3..eeea9cda2f4 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -963,6 +963,35 @@ def test_http(self):
self.assertEqual(req.unredirected_hdrs["Host"], "baz")
self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
+ def test_http_header_priority(self):
+ # gh-47005: regular headers set via add_header() must override
+ # unredirected headers with the same name in do_open(), consistent
+ # with get_header() and header_items().
+ cases = [
+ ("Content-Type", "application/json", "application/x-www-form-urlencoded"),
+ ("Content-Length", "99", "0"),
+ ("Host", "override.example.com", "internal.example.com"),
+ ("Authorization", "Bearer user-token", "Basic stale="),
+ ("Cookie", "a=1", "b=2"),
+ ("User-Agent", "MyApp/1.0", "Python-urllib/test"),
+ ]
+ h = urllib.request.AbstractHTTPHandler()
+ h.parent = MockOpener()
+
+ for key, regular, unredirected in cases:
+ req = Request("http://example.com/", headers={key: regular})
+ req.timeout = None
+ req.add_unredirected_header(key, unredirected)
+
+ http = MockHTTPClass()
+ h.do_open(http, req)
+
+ sent_headers = dict(http.req_headers)
+ self.assertEqual(sent_headers[key], regular)
+ # key is capitalized by add_header() and add_unredirected_header() calls
+ self.assertEqual(req.get_header(key.capitalize()), regular)
+ self.assertEqual(dict(req.header_items())[key.capitalize()], regular)
+
def test_http_body_file(self):
# A regular file - chunked encoding is used unless Content Length is
# already set.
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
index b2bde5a9b1d..98f0b190ee7 100644
--- a/Lib/test/test_urlparse.py
+++ b/Lib/test/test_urlparse.py
@@ -1426,15 +1426,20 @@ def test_splitting_bracketed_hosts(self):
self.assertEqual(p1.username, 'user')
self.assertEqual(p1.path, '/path')
self.assertEqual(p1.port, 1234)
- p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query')
- self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test')
+ p2 = urllib.parse.urlsplit('scheme://user@[V6a.ip]:1234/path?query')
+ self.assertEqual(p2.hostname, 'v6a.ip')
self.assertEqual(p2.username, 'user')
self.assertEqual(p2.path, '/path')
- self.assertIs(p2.port, None)
- p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query')
- self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test')
+ self.assertEqual(p2.port, 1234)
+ p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query')
+ self.assertEqual(p3.hostname, '0439:23af:2309::fae7%test')
self.assertEqual(p3.username, 'user')
self.assertEqual(p3.path, '/path')
+ self.assertIs(p3.port, None)
+ p4 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query')
+ self.assertEqual(p4.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test')
+ self.assertEqual(p4.username, 'user')
+ self.assertEqual(p4.path, '/path')
def test_port_casting_failure_message(self):
message = "Port could not be cast to integer value as 'oracle'"
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index a651e815ddc..c9f8a33e72c 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -461,8 +461,8 @@ def _check_bracketed_netloc(netloc):
# Valid bracketed hosts are defined in
# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/
def _check_bracketed_host(hostname):
- if hostname.startswith('v'):
- if not re.match(r"\Av[a-fA-F0-9]+\..+\z", hostname):
+ if hostname.startswith(('v', 'V')):
+ if not re.match(r"\A[vV][a-fA-F0-9]+\..+\z", hostname):
raise ValueError(f"IPvFuture address is invalid")
else:
ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 8d7470a2273..a8f40100b7b 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -1291,8 +1291,7 @@ def do_open(self, http_class, req, **http_conn_args):
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
- headers.update({k: v for k, v in req.headers.items()
- if k not in headers})
+ headers.update(req.headers)
# TODO(jhylton): Should this be redesigned to handle
# persistent connections?
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index e70eae80036..d267ed00345 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -65,9 +65,17 @@ def read(self):
f = urllib.request.urlopen(self.url)
except urllib.error.HTTPError as err:
if err.code in (401, 403):
+ # If access to robot.txt has the status Unauthorized/Forbidden,
+ # then most likely this applies to the entire site.
self.disallow_all = True
- elif err.code >= 400 and err.code < 500:
+ elif 400 <= err.code < 500:
+ # RFC 9309, Section 2.3.1.3: the crawler MAY access any
+ # resources on the server.
self.allow_all = True
+ elif 500 <= err.code < 600:
+ # RFC 9309, Section 2.3.1.4: the crawler MUST assume
+ # complete disallow.
+ self.disallow_all = True
err.close()
else:
raw = f.read()
@@ -135,15 +143,15 @@ def parse(self, lines):
# before trying to convert to int we need to make
# sure that robots.txt has valid syntax otherwise
# it will crash
- if line[1].strip().isdigit():
+ if line[1].strip().isdecimal():
entry.delay = int(line[1])
state = 2
elif line[0] == "request-rate":
if state != 0:
numbers = line[1].split('/')
# check if all values are sane
- if (len(numbers) == 2 and numbers[0].strip().isdigit()
- and numbers[1].strip().isdigit()):
+ if (len(numbers) == 2 and numbers[0].strip().isdecimal()
+ and numbers[1].strip().isdecimal()):
entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
state = 2
elif line[0] == "sitemap":
diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs
index db11e0a339f..f018dd32f93 100644
--- a/crates/common/src/cformat.rs
+++ b/crates/common/src/cformat.rs
@@ -55,6 +55,12 @@ impl fmt::Display for CFormatError {
pub type CFormatConversion = super::format::FormatConversion;
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CFormatContext {
+ Str,
+ Bytes,
+}
+
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u8)]
pub enum CNumberType {
@@ -98,6 +104,7 @@ pub enum CFormatType {
Float(CFloatType),
Character(CCharacterType),
String(CFormatConversion),
+ Bytes,
}
impl CFormatType {
@@ -108,6 +115,7 @@ impl CFormatType {
Self::Float(x) => x as u8 as char,
Self::Character(x) => x as u8 as char,
Self::String(x) => x as u8 as char,
+ Self::Bytes => 'b',
}
}
}
@@ -296,14 +304,14 @@ impl FromStr for CFormatSpecKeyed {
return Err((CFormatErrorType::MissingModuloSign, 1));
}
- Self::parse(&mut chars)
+ Self::parse(&mut chars, CFormatContext::Str)
}
}
pub type ParseIter = Peekable>;
impl CFormatSpecKeyed {
- pub fn parse(iter: &mut ParseIter) -> Result
+ pub fn parse(iter: &mut ParseIter, context: CFormatContext) -> Result
where
I: Iterator- ,
{
@@ -313,7 +321,7 @@ impl CFormatSpecKeyed {
parse_quantity(iter, isize::MAX as usize, CFormatErrorType::WidthTooBig)?;
let precision = parse_precision(iter)?;
consume_length(iter);
- let format_type = parse_format_type(iter)?;
+ let format_type = parse_format_type(iter, context)?;
let spec = CFormatSpec {
flags,
@@ -618,7 +626,10 @@ where
iter.next_if(|(_, c)| matches!(c.to_char_lossy(), 'h' | 'l' | 'L'));
}
-fn parse_format_type(iter: &mut ParseIter) -> Result
+fn parse_format_type(
+ iter: &mut ParseIter,
+ context: CFormatContext,
+) -> Result
where
C: FormatChar,
I: Iterator
- ,
@@ -646,8 +657,8 @@ where
'c' => CFormatType::Character(CCharacterType::Character),
'r' => CFormatType::String(CFormatConversion::Repr),
's' => CFormatType::String(CFormatConversion::Str),
- 'b' => CFormatType::String(CFormatConversion::Bytes),
'a' => CFormatType::String(CFormatConversion::Ascii),
+ 'b' if context == CFormatContext::Bytes => CFormatType::Bytes,
_ => return Err((CFormatErrorType::UnsupportedFormatChar(c.into()), index)),
})
}
@@ -784,7 +795,7 @@ impl
CFormatStrOrBytes {
self.parts.iter_mut()
}
- pub fn parse(iter: &mut ParseIter) -> Result
+ pub fn parse(iter: &mut ParseIter, context: CFormatContext) -> Result
where
S: FormatBuf,
I: Iterator- ,
@@ -808,10 +819,11 @@ impl
CFormatStrOrBytes {
));
}
- let spec = CFormatSpecKeyed::parse(iter).map_err(|err| CFormatError {
- typ: err.0,
- index: err.1,
- })?;
+ let spec =
+ CFormatSpecKeyed::parse(iter, context).map_err(|err| CFormatError {
+ typ: err.0,
+ index: err.1,
+ })?;
parts.push((index, CFormatPart::Spec(spec)));
if let Some(&(index, _)) = iter.peek() {
@@ -848,7 +860,7 @@ pub type CFormatBytes = CFormatStrOrBytes>;
impl CFormatBytes {
pub fn parse_from_bytes(bytes: &[u8]) -> Result {
let mut iter = bytes.iter().copied().enumerate().peekable();
- Self::parse(&mut iter)
+ Self::parse(&mut iter, CFormatContext::Bytes)
}
}
@@ -859,7 +871,7 @@ impl FromStr for CFormatString {
fn from_str(text: &str) -> Result {
let mut iter = text.chars().enumerate().peekable();
- Self::parse(&mut iter)
+ Self::parse(&mut iter, CFormatContext::Str)
}
}
@@ -868,7 +880,7 @@ pub type CFormatWtf8 = CFormatStrOrBytes;
impl CFormatWtf8 {
pub fn parse_from_wtf8(s: &Wtf8) -> Result {
let mut iter = s.code_points().enumerate().peekable();
- Self::parse(&mut iter)
+ Self::parse(&mut iter, CFormatContext::Str)
}
}
diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs
index 7a5e3ac0fa1..0308721975a 100644
--- a/crates/common/src/format.rs
+++ b/crates/common/src/format.rs
@@ -37,7 +37,6 @@ pub enum FormatConversion {
Str = b's',
Repr = b'r',
Ascii = b'b',
- Bytes = b'a',
}
impl FormatParse for FormatConversion {
@@ -59,7 +58,6 @@ impl FormatConversion {
's' => Some(Self::Str),
'r' => Some(Self::Repr),
'a' => Some(Self::Ascii),
- 'b' => Some(Self::Bytes),
_ => None,
}
}
diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs
index 7d47da39928..00147103851 100644
--- a/crates/vm/src/cformat.rs
+++ b/crates/vm/src/cformat.rs
@@ -32,38 +32,37 @@ fn spec_format_bytes(
obj: PyObjectRef,
) -> PyResult> {
match &spec.format_type {
- CFormatType::String(conversion) => match conversion {
- // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to
- // %a for strings (not %r)
- CFormatConversion::Repr | CFormatConversion::Ascii => {
- let b = builtins::ascii(obj, vm)?.as_bytes().to_vec();
- Ok(b)
+ // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to
+ // %a for strings (not %r)
+ CFormatType::String(CFormatConversion::Repr | CFormatConversion::Ascii) => {
+ let b = builtins::ascii(obj, vm)?.as_bytes().to_vec();
+ Ok(b)
+ }
+ // %b and %s are equivalent for bytes formatting.
+ // Mirrors CPython's format_obj() in bytesobject.c
+ CFormatType::Bytes | CFormatType::String(CFormatConversion::Str) => {
+ if let Some(bytes) = obj.downcast_ref::() {
+ return Ok(spec.format_bytes(bytes.as_bytes()));
}
- // format_obj
- CFormatConversion::Str | CFormatConversion::Bytes => {
- if let Some(bytes) = obj.downcast_ref::() {
- return Ok(spec.format_bytes(bytes.as_bytes()));
- }
- if let Some(bytearray) = obj.downcast_ref::() {
- return Ok(spec.format_bytes(&bytearray.borrow_buf()));
- }
- if let Some(method) = vm.get_special_method(&obj, identifier!(vm, __bytes__))? {
- let bytes = method.invoke((), vm)?;
- let bytes = PyBytes::try_from_borrowed_object(vm, &bytes)?;
- return Ok(spec.format_bytes(bytes.as_bytes()));
- }
- if obj.check_buffer() {
- let buffer = PyBuffer::from_object(vm, &obj, BufferFlags::FULL_RO)?;
- return Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes)));
- }
- let msg = format!(
- "%b requires a bytes-like object, or an object that \
- implements __bytes__, not '{}'",
- obj.class().name()
- );
- Err(vm.new_type_error(msg))
+ if let Some(bytearray) = obj.downcast_ref::() {
+ return Ok(spec.format_bytes(&bytearray.borrow_buf()));
}
- },
+ if let Some(method) = vm.get_special_method(&obj, identifier!(vm, __bytes__))? {
+ let bytes = method.invoke((), vm)?;
+ let bytes = PyBytes::try_from_borrowed_object(vm, &bytes)?;
+ return Ok(spec.format_bytes(bytes.as_bytes()));
+ }
+ if obj.check_buffer() {
+ let buffer = PyBuffer::from_object(vm, &obj, BufferFlags::FULL_RO)?;
+ return Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes)));
+ }
+ let msg = format!(
+ "%b requires a bytes-like object, or an object that \
+ implements __bytes__, not '{}'",
+ obj.class().name()
+ );
+ Err(vm.new_type_error(msg))
+ }
CFormatType::Number(number_type) => match number_type {
CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => {
match_class!(match &obj {
@@ -166,7 +165,6 @@ fn spec_format_string(
vm: &VirtualMachine,
spec: &CFormatSpec,
obj: PyObjectRef,
- idx: usize,
) -> PyResult {
match &spec.format_type {
CFormatType::String(conversion) => {
@@ -174,16 +172,13 @@ fn spec_format_string(
CFormatConversion::Ascii => builtins::ascii(obj, vm)?.as_wtf8().to_owned(),
CFormatConversion::Str => obj.str(vm)?.as_wtf8().to_owned(),
CFormatConversion::Repr => obj.repr(vm)?.as_wtf8().to_owned(),
- CFormatConversion::Bytes => {
- // idx is the position of the %, we want the position of the b
- return Err(vm.new_value_error(format!(
- "unsupported format character 'b' (0x62) at index {}",
- idx + 1
- )));
- }
};
Ok(spec.format_string(result))
}
+ CFormatType::Bytes => {
+ // 'b' is rejected at parse time in Str context, see `CFormatContext`.
+ unreachable!("%b cannot be parsed in a str format string")
+ }
CFormatType::Number(number_type) => match number_type {
CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => {
match_class!(match &obj {
@@ -487,12 +482,12 @@ pub(crate) fn cformat_string(
}
// dict
- for (idx, part) in format {
+ for (_, part) in format {
match part {
CFormatPart::Literal(literal) => result.push_wtf8(&literal),
CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => {
let value = values_obj.get_item(&mapping_key.unwrap(), vm)?;
- let part_result = spec_format_string(vm, &spec, value, idx)?;
+ let part_result = spec_format_string(vm, &spec, value)?;
result.push_wtf8(&part_result);
}
}
@@ -510,7 +505,7 @@ pub(crate) fn cformat_string(
let mut value_iter = values.iter();
- for (idx, part) in format {
+ for (_, part) in format {
match part {
CFormatPart::Literal(literal) => result.push_wtf8(&literal),
CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => {
@@ -526,7 +521,7 @@ pub(crate) fn cformat_string(
return Err(vm.new_type_error("not enough arguments for format string"));
};
- let part_result = spec_format_string(vm, &spec, value.clone(), idx)?;
+ let part_result = spec_format_string(vm, &spec, value.clone())?;
result.push_wtf8(&part_result);
}
}
diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs
index 75f5c32f0c3..de2391dcde5 100644
--- a/crates/vm/src/format.rs
+++ b/crates/vm/src/format.rs
@@ -153,9 +153,6 @@ fn format_internal(
Some(FormatConversion::Str) => argument.str(vm)?.into(),
Some(FormatConversion::Repr) => argument.repr(vm)?.into(),
Some(FormatConversion::Ascii) => builtins::ascii(argument, vm)?.into(),
- Some(FormatConversion::Bytes) => {
- vm.call_method(&argument, identifier!(vm, decode).as_str(), ())?
- }
None => {
return Err(
vm.new_value_error(format!("Unknown conversion specifier {c}"))
diff --git a/extra_tests/snippets/builtin_format.py b/extra_tests/snippets/builtin_format.py
index 6b7403da133..844d9e45138 100644
--- a/extra_tests/snippets/builtin_format.py
+++ b/extra_tests/snippets/builtin_format.py
@@ -66,6 +66,15 @@ def test_zero_padding():
else:
raise AssertionError("expected ValueError for unknown conversion specifier '!x'")
+# 'b' is a valid conversion specifier for %-style bytes formatting, but not for str.format().
+try:
+ "{0!b}".format(3)
+except ValueError as error:
+ if str(error) != "Unknown conversion specifier b":
+ raise AssertionError(f"unexpected error message: {error}") from error
+else:
+ raise AssertionError("expected ValueError for unknown conversion specifier '!b'")
+
assert "{:,}".format(100) == "100"
assert "{:,}".format(1024) == "1,024"
assert "{:_}".format(65536) == "65_536"