Fix collections.abc imports on Python 3.13.0 and 3.13.1 (#2657) (#2658)

* Fix `collections.abc` imports on Python 3.13.0 and 3.13.1 (#2657)

* Add test for importing `collections.abc`

* Fix issue with importing of frozen submodules

* Fix `search_paths`

* Do not reassign submodule_path parameters in method bodies

This makes it easier to use less generic annotations with mypy.

---------

Co-authored-by: Daniël van Noord <[email protected]>
Co-authored-by: correctmost <[email protected]>

Author: 3 people

ChangeLog

@@ -13,13 +13,18 @@ What's New in astroid 3.3.7?
 ============================
 Release date: TBA
 
+* Fix inability to import `collections.abc` in python 3.13.1. The reported fix in astroid 3.3.6
+  did not actually fix this issue.
+
+  Closes pylint-dev/pylint#10112
 
 
 What's New in astroid 3.3.6?
 ============================
 Release date: 2024-12-08
 
 * Fix inability to import `collections.abc` in python 3.13.1.
+  _It was later found that this did not resolve the linked issue. It was fixed in astroid 3.3.7_
 
   Closes pylint-dev/pylint#10112
 

astroid/brain/brain_collections.py

@@ -8,7 +8,7 @@
 
 from astroid.brain.helpers import register_module_extender
 from astroid.builder import AstroidBuilder, extract_node, parse
-from astroid.const import PY313_0, PY313_PLUS
+from astroid.const import PY313_PLUS
 from astroid.context import InferenceContext
 from astroid.exceptions import AttributeInferenceError
 from astroid.manager import AstroidManager
@@ -20,8 +20,7 @@
 
 def _collections_transform():
     return parse(
-        ("    import _collections_abc as abc" if PY313_PLUS and not PY313_0 else "")
-        + """
+        """
     class defaultdict(dict):
         default_factory = None
         def __missing__(self, key): pass
@@ -33,7 +32,7 @@ def __getitem__(self, key): return default_factory
     )
 
 
-def _collections_abc_313_0_transform() -> nodes.Module:
+def _collections_abc_313_transform() -> nodes.Module:
     """See https://github.com/python/cpython/pull/124735"""
     return AstroidBuilder(AstroidManager()).string_build(
         "from _collections_abc import *"
@@ -133,7 +132,7 @@ def register(manager: AstroidManager) -> None:
         ClassDef, easy_class_getitem_inference, _looks_like_subscriptable
     )
 
-    if PY313_0:
+    if PY313_PLUS:
         register_module_extender(
-            manager, "collections.abc", _collections_abc_313_0_transform
+            manager, "collections.abc", _collections_abc_313_transform
         )

astroid/const.py

@@ -9,7 +9,6 @@
 PY311_PLUS = sys.version_info >= (3, 11)
 PY312_PLUS = sys.version_info >= (3, 12)
 PY313_PLUS = sys.version_info >= (3, 13)
-PY313_0 = sys.version_info[:3] == (3, 13, 0)
 
 WIN32 = sys.platform == "win32"
 

astroid/interpreter/_import/spec.py

@@ -133,36 +133,66 @@ def find_module(
         processed: list[str],
         submodule_path: Sequence[str] | None,
     ) -> ModuleSpec | None:
-        if submodule_path is not None:
-            submodule_path = list(submodule_path)
-        elif modname in sys.builtin_module_names:
+        # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins.
+        # Therefore, we use the `builtin_module_nams` heuristic for these.
+        if submodule_path is None and modname in sys.builtin_module_names:
             return ModuleSpec(
                 name=modname,
                 location=None,
                 type=ModuleType.C_BUILTIN,
             )
-        else:
-            try:
-                with warnings.catch_warnings():
-                    warnings.filterwarnings("ignore", category=UserWarning)
-                    spec = importlib.util.find_spec(modname)
+
+        # sys.stdlib_module_names was added in Python 3.10
+        if PY310_PLUS:
+            # If the module is a stdlib module, check whether this is a frozen module. Note that
+            # `find_spec` actually imports the module, so we want to make sure we only run this code
+            # for stuff that can be expected to be frozen. For now this is only stdlib.
+            if modname in sys.stdlib_module_names or (
+                processed and processed[0] in sys.stdlib_module_names
+            ):
+                spec = importlib.util.find_spec(".".join((*processed, modname)))
                 if (
                     spec
                     and spec.loader  # type: ignore[comparison-overlap] # noqa: E501
                     is importlib.machinery.FrozenImporter
                 ):
-                    # No need for BuiltinImporter; builtins handled above
                     return ModuleSpec(
                         name=modname,
                         location=getattr(spec.loader_state, "filename", None),
                         type=ModuleType.PY_FROZEN,
                     )
-            except ValueError:
-                pass
-            submodule_path = sys.path
+        else:
+            # NOTE: This is broken code. It doesn't work on Python 3.13+ where submodules can also
+            # be frozen. However, we don't want to worry about this and we don't want to break
+            # support for older versions of Python. This is just copy-pasted from the old non
+            # working version to at least have no functional behaviour change on <=3.10.
+            # It can be removed after 3.10 is no longer supported in favour of the logic above.
+            if submodule_path is None:  # pylint: disable=else-if-used
+                try:
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings("ignore", category=UserWarning)
+                        spec = importlib.util.find_spec(modname)
+                    if (
+                        spec
+                        and spec.loader  # type: ignore[comparison-overlap] # noqa: E501
+                        is importlib.machinery.FrozenImporter
+                    ):
+                        # No need for BuiltinImporter; builtins handled above
+                        return ModuleSpec(
+                            name=modname,
+                            location=getattr(spec.loader_state, "filename", None),
+                            type=ModuleType.PY_FROZEN,
+                        )
+                except ValueError:
+                    pass
+
+        if submodule_path is not None:
+            search_paths = list(submodule_path)
+        else:
+            search_paths = sys.path
 
         suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0])
-        for entry in submodule_path:
+        for entry in search_paths:
             package_directory = os.path.join(entry, modname)
             for suffix in suffixes:
                 package_file_name = "__init__" + suffix
@@ -231,13 +261,12 @@ def find_module(
         if processed:
             modname = ".".join([*processed, modname])
         if util.is_namespace(modname) and modname in sys.modules:
-            submodule_path = sys.modules[modname].__path__
             return ModuleSpec(
                 name=modname,
                 location="",
                 origin="namespace",
                 type=ModuleType.PY_NAMESPACE,
-                submodule_search_locations=submodule_path,
+                submodule_search_locations=sys.modules[modname].__path__,
             )
         return None
 
@@ -353,13 +382,15 @@ def _search_zip(
             if PY310_PLUS:
                 if not importer.find_spec(os.path.sep.join(modpath)):
                     raise ImportError(
-                        "No module named %s in %s/%s"
-                        % (".".join(modpath[1:]), filepath, modpath)
+                        "No module named {} in {}/{}".format(
+                            ".".join(modpath[1:]), filepath, modpath
+                        )
                     )
             elif not importer.find_module(os.path.sep.join(modpath)):
                 raise ImportError(
-                    "No module named %s in %s/%s"
-                    % (".".join(modpath[1:]), filepath, modpath)
+                    "No module named {} in {}/{}".format(
+                        ".".join(modpath[1:]), filepath, modpath
+                    )
                 )
             return (
                 ModuleType.PY_ZIPMODULE,

tests/brain/test_brain.py

@@ -192,6 +192,22 @@ def check_metaclass_is_abc(node: nodes.ClassDef):
 
 
 class CollectionsBrain(unittest.TestCase):
+    def test_collections_abc_is_importable(self) -> None:
+        """
+        Test that we can import `collections.abc`.
+
+        The collections.abc has gone through various formats of being frozen. Therefore, we ensure
+        that we can still import it (correctly).
+        """
+        import_node = builder.extract_node("import collections.abc")
+        assert isinstance(import_node, nodes.Import)
+        imported_module = import_node.do_import_module(import_node.names[0][0])
+        # Make sure that the file we have imported is actually the submodule of collections and
+        # not the `abc` module. (Which would happen if you call `importlib.util.find_spec("abc")`
+        # instead of `importlib.util.find_spec("collections.abc")`)
+        assert isinstance(imported_module.file, str)
+        assert "collections" in imported_module.file
+
     def test_collections_object_not_subscriptable(self) -> None:
         """
         Test that unsubscriptable types are detected