Merge remote-tracking branch 'origin/main' into gh-158195-unpin-type-lock

Author: nascheme

Doc/library/ast.rst

@@ -48,10 +48,10 @@ Node classes
    module and re-exported in :mod:`!ast`.
 
    There is one class defined for each left-hand side symbol in the abstract
-   grammar (for example, :class:`ast.stmt` or :class:`ast.expr`).  In addition,
+   grammar (for example, :class:`!ast.stmt` or :class:`!ast.expr`).  In addition,
    there is one class defined for each constructor on the right-hand side; these
    classes inherit from the classes for the left-hand side trees.  For example,
-   :class:`ast.BinOp` inherits from :class:`ast.expr`.  For production rules
+   :class:`ast.BinOp` inherits from :class:`!ast.expr`.  For production rules
    with alternatives (aka "sums"), the left-hand side class is abstract: only
    instances of specific constructor nodes are ever created.
 
@@ -65,7 +65,7 @@ Node classes
 
       Each instance of a concrete class has one attribute for each child node,
       of the type as defined in the grammar.  For example, :class:`ast.BinOp`
-      instances have an attribute :attr:`left` of type :class:`ast.expr`.
+      instances have an attribute :attr:`!left` of type :class:`!ast.expr`.
 
       If these attributes are marked as optional in the grammar (using a
       question mark), the value might be ``None``.  If the attributes can have
@@ -104,10 +104,11 @@ Node classes
       one can get the source segment of a one-line expression node using
       ``source_line[node.col_offset : node.end_col_offset]``.
 
-   The constructor of a class :class:`ast.T` parses its arguments as follows:
+   The constructors of AST node classes parse their arguments as follows:
 
    * If there are positional arguments, there must be as many as there are items
-     in :attr:`T._fields`; they will be assigned as attributes of these names.
+     in :attr:`!_fields`; they will be assigned to attributes with
+     the corresponding names.
    * If there are keyword arguments, they will set the attributes of the same
      names to the given values.
 
@@ -2223,8 +2224,8 @@ Async and await
 
 .. note::
    When a string is parsed by :func:`ast.parse`, operator nodes (subclasses
-   of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`,
-   :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree
+   of :class:`!ast.operator`, :class:`!ast.unaryop`, :class:`!ast.cmpop`,
+   :class:`!ast.boolop` and :class:`!ast.expr_context`) on the returned tree
    will be singletons. Changes to one will be reflected in all other
    occurrences of the same value (for example, :class:`ast.Add`).
 

Lib/collections/__init__.py

@@ -1536,6 +1536,8 @@ def encode(self, encoding='utf-8', errors='strict'):
         return self.data.encode(encoding, errors)
 
     def endswith(self, suffix, start=0, end=_sys.maxsize):
+        if isinstance(suffix, UserString):
+            suffix = suffix.data
         return self.data.endswith(suffix, start, end)
 
     def expandtabs(self, tabsize=8):
@@ -1646,6 +1648,8 @@ def splitlines(self, keepends=False):
         return self.data.splitlines(keepends)
 
     def startswith(self, prefix, start=0, end=_sys.maxsize):
+        if isinstance(prefix, UserString):
+            prefix = prefix.data
         return self.data.startswith(prefix, start, end)
 
     def strip(self, chars=None):

Lib/test/test_clinic.py

@@ -15,14 +15,16 @@
 import re
 import sys
 import unittest
+from unittest import mock
 import warnings
 
 test_tools.skip_if_missing('clinic')
 with test_tools.imports_under_tool('clinic'):
     import libclinic
     from libclinic import ClinicError, unspecified, NULL, fail
     from libclinic.converters import (
-        int_converter, object_converter, str_converter, self_converter)
+        int_converter, object_converter, str_converter, self_converter,
+        Py_ssize_t_converter)
     from libclinic.function import (
         Module, Class, Function, FunctionKind, Parameter,
         permute_optional_groups, permute_right_option_groups,
@@ -34,6 +36,7 @@
     from libclinic.block_parser import Block, BlockParser
     from libclinic.codegen import BlockPrinter, Destination
     from libclinic.dsl_parser import DSLParser
+    from libclinic.slots import SlotSpec, SLOTS
     from libclinic.cli import parse_file, Clinic
 
 
@@ -807,6 +810,112 @@ def test_var_keyword_non_dict(self):
         """
         self.expect_failure(block, err, lineno=4)
 
+    def test_slot_next(self):
+        block = """
+            /*[clinic input]
+            output everything block
+            class Foo "FooObject *" "&Foo_Type"
+            [clinic start generated code]*/
+            /*[clinic input]
+            Foo.__next__
+            [clinic start generated code]*/
+            /*[clinic input]
+            @critical_section
+            Foo.__next__ as foo_iternext
+            [clinic start generated code]*/
+            #ifdef CONDITION
+            /*[clinic input]
+            Foo.__next__ as foo_iternext_cond
+            [clinic start generated code]*/
+            #endif
+        """
+        generated = self.clinic.parse(dedent(block))
+        self.assertIn(dedent("""
+            static PyObject *
+            Foo_next(PyObject *self)
+            {
+                return Foo_next_impl((FooObject *)self);
+            }
+        """), generated)
+        self.assertIn(dedent("""
+            static PyObject *
+            foo_iternext(PyObject *self)
+            {
+                PyObject *return_value = NULL;
+
+                Py_BEGIN_CRITICAL_SECTION(self);
+                return_value = foo_iternext_impl((FooObject *)self);
+                Py_END_CRITICAL_SECTION();
+
+                return return_value;
+            }
+        """), generated)
+        self.assertIn("#if defined(CONDITION)\n\nstatic PyObject *\n"
+                      "foo_iternext_cond_impl(FooObject *self);", generated)
+        # A slot function is not stored in a PyMethodDef.
+        for text in "METHODDEF", "PyDoc_", "#ifndef":
+            self.assertNotIn(text, generated)
+
+    def test_slot_with_parameters_and_return_type(self):
+        # Slots which take arguments or return something else than an
+        # object are not supported yet; check the machinery with fake ones.
+        slots = {
+            '__contains__': SlotSpec('__contains__', 'int', 'int',
+                                     {(object_converter,): 'sq_contains'}),
+            '__bool__': SlotSpec('__bool__', 'int', 'int', {(): 'nb_bool'}),
+        }
+        block = """
+            /*[clinic input]
+            output everything block
+            class Foo "FooObject *" "&Foo_Type"
+            [clinic start generated code]*/
+            /*[clinic input]
+            Foo.__contains__
+                value: object
+                /
+            [clinic start generated code]*/
+            /*[clinic input]
+            @critical_section
+            Foo.__bool__
+            [clinic start generated code]*/
+        """
+        with mock.patch.dict(SLOTS, slots):
+            generated = self.clinic.parse(dedent(block))
+        self.assertIn(dedent("""
+            static int
+            Foo_contains(PyObject *self, PyObject *value)
+            {
+                return Foo_contains_impl((FooObject *)self, value);
+            }
+        """), generated)
+        self.assertIn(dedent("""
+            static int
+            Foo_bool(PyObject *self)
+            {
+                int return_value = -1;
+
+                Py_BEGIN_CRITICAL_SECTION(self);
+                return_value = Foo_bool_impl((FooObject *)self);
+                Py_END_CRITICAL_SECTION();
+
+                return return_value;
+            }
+        """), generated)
+
+    def test_slot_cannot_be_cloned(self):
+        block = """
+            /*[clinic input]
+            class Foo "FooObject *" "&Foo_Type"
+            [clinic start generated code]*/
+            /*[clinic input]
+            Foo.__next__
+            [clinic start generated code]*/
+            /*[clinic input]
+            Foo.other = Foo.__next__
+            [clinic start generated code]*/
+        """
+        self.expect_failure(block, "Type slot methods cannot be cloned", lineno=8)
+
     def test_getset_in_ifdef(self):
         block = """
             /*[clinic input]
@@ -3237,6 +3346,78 @@ class Foo "" ""
                                   f"a normal callable")
                 self.expect_failure(block, expected_error, lineno=3)
 
+    def test_slot_next(self):
+        function = self.parse_function("""
+            module foo
+            class Foo "FooObject *" ""
+            Foo.__next__
+        """, signatures_in_block=3, function_index=2)
+        self.assertIs(function.kind, FunctionKind.SLOT)
+        self.assertEqual(function.slot.name, "__next__")
+        self.assertEqual(function.c_basename, "Foo_next")
+        self.assertEqual(function.return_converter.type, "PyObject *")
+        self.assertEqual(list(function.parameters), ["self"])
+
+    def test_slot_parameters(self):
+        slot = SlotSpec('__contains__', 'int', 'int', {
+            (Py_ssize_t_converter,): 'sq_fake',
+            (object_converter,): 'sq_contains'})
+        block = """
+            module foo
+            class Foo "FooObject *" ""
+            Foo.__contains__
+                value: {converter}
+                /
+        """
+        errors = [
+            ("Foo.__contains__",
+             "'__contains__' must declare (Py_ssize_t) for sq_fake or "
+             "(object) for sq_contains; got ()"),
+            ("Foo.__contains__\n    value: int\n    /",
+             "'__contains__' must declare"),
+            ("Foo.__contains__\n    value: object",
+             "The parameters of '__contains__' must be positional-only"),
+            ("Foo.__contains__\n    value: object = None\n    /",
+             "The parameters of '__contains__' cannot have a default value"),
+        ]
+        with mock.patch.dict(SLOTS, {'__contains__': slot}):
+            for converter in "Py_ssize_t", "object":
+                with self.subTest(converter=converter):
+                    function = self.parse_function(
+                        block.format(converter=converter),
+                        signatures_in_block=3, function_index=2)
+                    self.assertIs(function.kind, FunctionKind.SLOT)
+                    self.assertEqual(function.return_converter.type, "int")
+                    self.assertEqual(list(function.parameters), ["self", "value"])
+            for body, err in errors:
+                with self.subTest(block=body):
+                    body = 'class Foo "FooObject *" "&Foo_Type"\n' + body
+                    self.expect_failure(body, err)
+
+    def test_slot_errors(self):
+        cases = [
+            ("module foo\nfoo.__next__",
+             "'__next__' must be a method!"),
+            ("@classmethod\nFoo.__next__",
+             "'__next__' must be a normal method"),
+            ("@getter\nFoo.__next__",
+             "'__next__' must be a normal method"),
+            ("@coexist\nFoo.__next__",
+             "@coexist cannot be applied to '__next__'"),
+            ("@vectorcall\nFoo.__next__",
+             "@vectorcall can only be used with __init__ and __new__"),
+            ("Foo.__next__\n    a: int",
+             "'__next__' methods cannot define parameters"),
+            ("Foo.__next__ -> int",
+             "'Foo.__next__' cannot define a return type"),
+            ("Foo.__next__\n\nReturn the next item.",
+             "'__next__' methods cannot have a docstring"),
+        ]
+        for block, err in cases:
+            with self.subTest(block=block):
+                block = 'class Foo "FooObject *" "&Foo_Type"\n' + block
+                self.expect_failure(block, err)
+
     def test_deleter_without_setter(self):
         block = """
             module foo

Lib/test/test_getpath.py

@@ -407,6 +407,74 @@ def test_venv_changed_name_posix(self):
         actual = getpath(ns, expected)
         self.assertEqual(expected, actual)
 
+    def test_venv_relocated_base_installation_posix(self):
+        "Test a venv whose base installation was relocated from PREFIX."
+        ns = MockPosixNamespace(
+            argv0="/venv/bin/python",
+            PREFIX="/build/install",
+        )
+        ns.add_known_xfile("/relocated/bin/python3")
+        ns.add_known_xfile("/public/bin/python")
+        ns.add_known_link("/public/bin/python", "/relocated/bin/python3")
+        ns.add_known_xfile("/venv/bin/python")
+        ns.add_known_link("/venv/bin/python", "/public/bin/python")
+        ns.add_known_file("/relocated/lib/python9.8/os.py")
+        ns.add_known_dir("/relocated/lib/python9.8/lib-dynload")
+        ns.add_known_file("/venv/pyvenv.cfg", [
+            "home = /public/bin",
+        ])
+        expected = dict(
+            executable="/venv/bin/python",
+            prefix="/venv",
+            exec_prefix="/venv",
+            base_executable="/relocated/bin/python3",
+            base_prefix="/relocated",
+            base_exec_prefix="/relocated",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/relocated/lib/python98.zip",
+                "/relocated/lib/python9.8",
+                "/relocated/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_symlink_with_misleading_landmarks_posix(self):
+        "Test that landmarks near an external symlink do effect resolution."
+        ns = MockPosixNamespace(
+            argv0="/venv/bin/python",
+            PREFIX="/opt/python",
+        )
+        ns.add_known_xfile("/opt/python/bin/python3")
+        ns.add_known_xfile("/public/bin/python")
+        ns.add_known_link("/public/bin/python", "/opt/python/bin/python3")
+        ns.add_known_xfile("/venv/bin/python")
+        ns.add_known_link("/venv/bin/python", "/public/bin/python")
+        ns.add_known_file("/opt/python/lib/python9.8/os.py")
+        ns.add_known_dir("/opt/python/lib/python9.8/lib-dynload")
+        ns.add_known_file("/public/lib/python9.8/os.py")
+        ns.add_known_dir("/public/lib/python9.8/lib-dynload")
+        ns.add_known_file("/venv/pyvenv.cfg", [
+            "home = /public/bin",
+        ])
+        expected = dict(
+            executable="/venv/bin/python",
+            prefix="/venv",
+            exec_prefix="/venv",
+            base_executable="/opt/python/bin/python3",
+            base_prefix="/opt/python",
+            base_exec_prefix="/opt/python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/opt/python/lib/python98.zip",
+                "/opt/python/lib/python9.8",
+                "/opt/python/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
     def test_venv_non_installed_zip_path_posix(self):
         "Test a venv created from non-installed python has correct zip path."""
         ns = MockPosixNamespace(