Remove dependency on pkg_resources from setuptools (#1536)

Avoid using `importlib.util.find_spec()` to avoid actually importing parent packages.

Author: jacobtylerwalls

ChangeLog

@@ -15,6 +15,10 @@ Release date: TBA
 
  Closes #1512
 
+* Remove dependency on ``pkg_resources`` from ``setuptools``.
+
+ Closes #1103
+
 * Allowed ``AstroidManager.clear_cache`` to reload necessary brain plugins.
 
 * Fixed incorrect inferences after rebuilding the builtins module, e.g. by calling

astroid/interpreter/_import/spec.py

@@ -10,6 +10,7 @@
 import importlib.machinery
 import importlib.util
 import os
+import pathlib
 import sys
 import zipimport
 from collections.abc import Sequence
@@ -147,7 +148,7 @@ def contribute_to_path(self, spec, processed):
  # Builtin.
  return None
 
- if _is_setuptools_namespace(spec.location):
+ if _is_setuptools_namespace(Path(spec.location)):
  # extend_path is called, search sys.path for module/packages
  # of this name see pkgutil.extend_path documentation
  path = [
@@ -179,7 +180,7 @@ def contribute_to_path(self, spec, processed):
 
 
 class ExplicitNamespacePackageFinder(ImportlibFinder):
- """A finder for the explicit namespace packages, generated through pkg_resources."""
+ """A finder for the explicit namespace packages."""
 
  def find_module(self, modname, module_parts, processed, submodule_path):
  if processed:
@@ -253,12 +254,12 @@ def contribute_to_path(self, spec, processed):
 )
 
 
-def _is_setuptools_namespace(location):
+def _is_setuptools_namespace(location: pathlib.Path) -> bool:
  try:
- with open(os.path.join(location, "__init__.py"), "rb") as stream:
+ with open(location / "__init__.py", "rb") as stream:
  data = stream.read(4096)
  except OSError:
- return None
+ return False
  else:
  extend_path = b"pkgutil" in data and b"extend_path" in data
  declare_namespace = (

astroid/interpreter/_import/util.py

@@ -2,15 +2,35 @@
 # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE
 # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt
 
-try:
- import pkg_resources
-except ImportError:
- pkg_resources = None # type: ignore[assignment]
+import sys
+from functools import lru_cache
+from importlib.util import _find_spec_from_path
 
 
-def is_namespace(modname):
- return (
- pkg_resources is not None
- and hasattr(pkg_resources, "_namespace_packages")
- and modname in pkg_resources._namespace_packages
- )
+@lru_cache(maxsize=4096)
+def is_namespace(modname: str) -> bool:
+ if modname in sys.builtin_module_names:
+ return False
+
+ found_spec = None
+
+ # find_spec() attempts to import parent packages when given dotted paths.
+ # That's unacceptable here, so we fallback to _find_spec_from_path(), which does
+ # not, but requires instead that each single parent ('astroid', 'nodes', etc.)
+ # be specced from left to right.
+ processed_components = []
+ last_parent = None
+ for component in modname.split("."):
+ processed_components.append(component)
+ working_modname = ".".join(processed_components)
+ try:
+ found_spec = _find_spec_from_path(working_modname, last_parent)
+ except ValueError:
+ # executed .pth files may not have __spec__
+ return True
+ last_parent = working_modname
+
+ if found_spec is None:
+ return False
+
+ return found_spec.origin is None

astroid/manager.py

@@ -18,7 +18,7 @@
 
 from astroid.const import BRAIN_MODULES_DIRECTORY
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
-from astroid.interpreter._import import spec
+from astroid.interpreter._import import spec, util
 from astroid.modutils import (
  NoSourceFile,
  _cache_normalize_path_,
@@ -384,6 +384,7 @@ def clear_cache(self) -> None:
  for lru_cache in (
  LookupMixIn.lookup,
  _cache_normalize_path_,
+ util.is_namespace,
  ObjectModel.attributes,
  ):
  lru_cache.cache_clear()

setup.cfg

@@ -39,7 +39,6 @@ packages = find:
 install_requires =
  lazy_object_proxy>=1.4.0
  wrapt>=1.11,<2
- setuptools>=20.0
  typed-ast>=1.4.0,<2.0;implementation_name=="cpython" and python_version<"3.8"
  typing-extensions>=3.10;python_version<"3.10"
 python_requires = >=3.7.2

tests/unittest_manager.py

@@ -10,12 +10,11 @@
 from collections.abc import Iterator
 from contextlib import contextmanager
 
-import pkg_resources
-
 import astroid
 from astroid import manager, test_utils
 from astroid.const import IS_JYTHON
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
+from astroid.interpreter._import import util
 from astroid.modutils import is_standard_module
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef
@@ -111,6 +110,16 @@ def test_ast_from_namespace_pkgutil(self) -> None:
  def test_ast_from_namespace_pkg_resources(self) -> None:
  self._test_ast_from_old_namespace_package_protocol("pkg_resources")
 
+ def test_identify_old_namespace_package_protocol(self) -> None:
+ # Like the above cases, this package follows the old namespace package protocol
+ # astroid currently assumes such packages are in sys.modules, so import it
+ # pylint: disable-next=import-outside-toplevel
+ import tests.testdata.python3.data.path_pkg_resources_1.package.foo as _ # noqa
+
+ self.assertTrue(
+ util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1")
+ )
+
  def test_implicit_namespace_package(self) -> None:
  data_dir = os.path.dirname(resources.find("data/namespace_pep_420"))
  contribute = os.path.join(data_dir, "contribute_to_namespace")
@@ -131,7 +140,6 @@ def test_implicit_namespace_package(self) -> None:
  def test_namespace_package_pth_support(self) -> None:
  pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
  site.addpackage(resources.RESOURCE_PATH, pth, [])
- pkg_resources._namespace_packages["foogle"] = []
 
  try:
  module = self.manager.ast_from_module_name("foogle.fax")
@@ -141,18 +149,14 @@ def test_namespace_package_pth_support(self) -> None:
  with self.assertRaises(AstroidImportError):
  self.manager.ast_from_module_name("foogle.moogle")
  finally:
- del pkg_resources._namespace_packages["foogle"]
  sys.modules.pop("foogle")
 
  def test_nested_namespace_import(self) -> None:
  pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
  site.addpackage(resources.RESOURCE_PATH, pth, [])
- pkg_resources._namespace_packages["foogle"] = ["foogle.crank"]
- pkg_resources._namespace_packages["foogle.crank"] = []
  try:
  self.manager.ast_from_module_name("foogle.crank")
  finally:
- del pkg_resources._namespace_packages["foogle"]
  sys.modules.pop("foogle")
 
  def test_namespace_and_file_mismatch(self) -> None:
@@ -161,12 +165,10 @@ def test_namespace_and_file_mismatch(self) -> None:
  self.assertEqual(ast.name, "unittest")
  pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
  site.addpackage(resources.RESOURCE_PATH, pth, [])
- pkg_resources._namespace_packages["foogle"] = []
  try:
  with self.assertRaises(AstroidImportError):
  self.manager.ast_from_module_name("unittest.foogle.fax")
  finally:
- del pkg_resources._namespace_packages["foogle"]
  sys.modules.pop("foogle")
 
  def _test_ast_from_zip(self, archive: str) -> None:
@@ -323,6 +325,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
  lrus = (
  astroid.nodes.node_classes.LookupMixIn.lookup,
  astroid.modutils._cache_normalize_path_,
+ util.is_namespace,
  astroid.interpreter.objectmodel.ObjectModel.attributes,
  )
 
@@ -332,6 +335,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
  # Generate some hits and misses
  ClassDef().lookup("garbage")
  is_standard_module("unittest", std_path=["garbage_path"])
+ util.is_namespace("unittest")
  astroid.interpreter.objectmodel.ObjectModel().attributes()
 
  # Did the hits or misses actually happen?