feat: Implement force_inspection option in the loader API

Author: pawamoy

src/griffe/loader.py

@@ -29,6 +29,7 @@
 from griffe.extensions.base import Extensions, load_extensions
 from griffe.finder import ModuleFinder, NamespacePackage, Package
 from griffe.git import tmp_worktree
+from griffe.importer import dynamic_import
 from griffe.logger import get_logger
 from griffe.merger import merge_stubs
 from griffe.stats import stats
@@ -55,6 +56,7 @@ def __init__(
  lines_collection: LinesCollection | None = None,
  modules_collection: ModulesCollection | None = None,
  allow_inspection: bool = True,
+ force_inspection: bool = False,
  store_source: bool = True,
  ) -> None:
  """Initialize the loader.
@@ -81,6 +83,8 @@ def __init__(
  """Collection of modules."""
  self.allow_inspection: bool = allow_inspection
  """Whether to allow inspecting (importing) modules for which we can't find sources."""
+ self.force_inspection: bool = force_inspection
+ """Whether to force inspecting (importing) modules, even when sources were found."""
  self.store_source: bool = store_source
  """Whether to store source code in the lines collection."""
  self.finder: ModuleFinder = ModuleFinder(search_paths)
@@ -159,47 +163,69 @@ def load(
  # TODO: Remove at some point.
  if objspec is None and module is None:
  raise TypeError("load() missing 1 required positional argument: 'objspec'")
+
  if objspec is None:
  objspec = module
  warnings.warn(
  "Parameter 'module' was renamed 'objspec' and made positional-only.",
  DeprecationWarning,
  stacklevel=2,
  )
+
  obj_path: str
- if objspec in _builtin_modules:
- logger.debug(f"{objspec} is a builtin module")
- if self.allow_inspection:
- logger.debug(f"Inspecting {objspec}")
- obj_path = objspec # type: ignore[assignment]
- top_module = self._inspect_module(objspec) # type: ignore[arg-type]
- self.modules_collection.set_member(top_module.path, top_module)
- obj = self.modules_collection.get_member(obj_path)
- self.extensions.call("on_package_loaded", pkg=obj)
- return obj
- raise LoadingError("Cannot load builtin module without inspection")
+ package = None
+ top_module = None
+
+ # We always start by searching paths on the disk,
+ # even if inspection is forced.
+ logger.debug(f"Searching path(s) for {objspec}")
  try:
  obj_path, package = self.finder.find_spec(
  objspec, # type: ignore[arg-type]
  try_relative_path=try_relative_path,
  find_stubs_package=find_stubs_package,
  )
  except ModuleNotFoundError:
- logger.debug(f"Could not find {objspec}")
- if self.allow_inspection:
- logger.debug(f"Trying inspection on {objspec}")
- obj_path = objspec # type: ignore[assignment]
- top_module = self._inspect_module(objspec) # type: ignore[arg-type]
- self.modules_collection.set_member(top_module.path, top_module)
- else:
+ # If we couldn't find paths on disk and inspection is disabled,
+ # re-raise ModuleNotFoundError.
+ logger.debug(f"Could not find path for {objspec} on disk")
+ if not (self.allow_inspection or self.force_inspection):
  raise
- else:
- logger.debug(f"Found {objspec}: loading")
+
+ # Otherwise we try to dynamically import the top-level module.
+ obj_path = str(objspec)
+ top_module_name = obj_path.split(".", 1)[0]
+ logger.debug(f"Trying to dynamically import {top_module_name}")
+ top_module_object = dynamic_import(top_module_name, self.finder.search_paths)
+
  try:
- top_module = self._load_package(package, submodules=submodules)
- except LoadingError as error:
- logger.exception(str(error)) # noqa: TRY401
- raise
+ top_module_path = top_module_object.__path__
+ except AttributeError:
+ # If the top-level module has no `__path__`, we inspect it as-is,
+ # and do not try to recurse into submodules (there shouldn't be any in builtin/compiled modules).
+ logger.debug(f"Module {top_module_name} has no paths set (built-in module?). Inspecting it as-is.")
+ top_module = self._inspect_module(top_module_name)
+ self.modules_collection.set_member(top_module.path, top_module)
+ obj = self.modules_collection.get_member(obj_path)
+ self.extensions.call("on_package_loaded", pkg=obj)
+ return obj
+
+ # We found paths, and use them to build our intermediate Package or NamespacePackage struct.
+ logger.debug(f"Module {top_module_name} has paths set: {top_module_path}")
+ if len(top_module_path) > 1:
+ package = NamespacePackage(top_module_name, top_module_path)
+ else:
+ package = Package(top_module_name, top_module_path[0])
+
+ # We have an intermediate package, and an object path: we're ready to load.
+ logger.debug(f"Found {objspec}: loading")
+ try:
+ top_module = self._load_package(package, submodules=submodules)
+ except LoadingError as error:
+ logger.exception(str(error)) # noqa: TRY401
+ raise
+
+ # Package is loaded, we now retrieve the initially requested object and return it.
  obj = self.modules_collection.get_member(obj_path)
  self.extensions.call("on_package_loaded", pkg=obj)
  return obj
@@ -534,9 +560,10 @@ def _load_module_path(
  logger.debug(f"Loading path {module_path}")
  if isinstance(module_path, list):
  module = self._create_module(module_name, module_path)
+ elif self.force_inspection:
+ module = self._inspect_module(module_name, module_path, parent)
  elif module_path.suffix in {".py", ".pyi"}:
- code = module_path.read_text(encoding="utf8")
- module = self._visit_module(code, module_name, module_path, parent)
+ module = self._visit_module(module_name, module_path, parent)
  elif self.allow_inspection:
  module = self._inspect_module(module_name, module_path, parent)
  else:
@@ -559,7 +586,7 @@ def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Pa
  except UnimportableModuleError as error:
  # NOTE: Why don't we load submodules when there's no init module in their folder?
  # Usually when a folder with Python files does not have an __init__.py module,
- # it's because the Python files are scripts, that should never be imported.
+ # it's because the Python files are scripts that should never be imported.
  # Django has manage.py somewhere for example, in a folder without init module.
  # This script isn't part of the Python API, as it's meant to be called on the CLI exclusively
  # (at least it was the case a few years ago when I was still using Django).
@@ -590,11 +617,13 @@ def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Pa
  logger.debug(str(error))
  else:
  if submodule_name in parent_module.members:
- logger.debug(
- f"Submodule '{submodule.path}' is shadowing the member at the same path. "
- "We recommend renaming the member or the submodule (for example prefixing it with `_`), "
- "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.",
- )
+ member = parent_module.members[submodule_name]
+ if member.is_alias or not member.is_module:
+ logger.debug(
+ f"Submodule '{submodule.path}' is shadowing the member at the same path. "
+ "We recommend renaming the member or the submodule (for example prefixing it with `_`), "
+ "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.",
+ )
  parent_module.set_member(submodule_name, submodule)
 
  def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module:
@@ -605,7 +634,8 @@ def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Mo
  modules_collection=self.modules_collection,
  )
 
- def _visit_module(self, code: str, module_name: str, module_path: Path, parent: Module | None = None) -> Module:
+ def _visit_module(self, module_name: str, module_path: Path, parent: Module | None = None) -> Module:
+ code = module_path.read_text(encoding="utf8")
  if self.store_source:
  self.lines_collection[module_path] = code.splitlines(keepends=False)
  start = datetime.now(tz=timezone.utc)
@@ -628,6 +658,8 @@ def _inspect_module(self, module_name: str, filepath: Path | None = None, parent
  for prefix in self.ignored_modules:
  if module_name.startswith(prefix):
  raise ImportError(f"Ignored module '{module_name}'")
+ if self.store_source and filepath and filepath.suffix in {".py", ".pyi"}:
+ self.lines_collection[filepath] = filepath.read_text(encoding="utf8").splitlines(keepends=False)
  start = datetime.now(tz=timezone.utc)
  try:
  module = inspect(
@@ -699,6 +731,7 @@ def load(
  lines_collection: LinesCollection | None = None,
  modules_collection: ModulesCollection | None = None,
  allow_inspection: bool = True,
+ force_inspection: bool = False,
  store_source: bool = True,
  find_stubs_package: bool = False,
  # TODO: Remove at some point.
@@ -739,6 +772,7 @@ def load(
  lines_collection: A collection of source code lines.
  modules_collection: A collection of modules.
  allow_inspection: Whether to allow inspecting modules when visiting them is not possible.
+ force_inspection: Whether to force using dynamic analysis when loading data.
  store_source: Whether to store code source in the lines collection.
  find_stubs_package: Whether to search for stubs-only package.
  If both the package and its stubs are found, they'll be merged together.
@@ -761,6 +795,7 @@ def load(
  lines_collection=lines_collection,
  modules_collection=modules_collection,
  allow_inspection=allow_inspection,
+ force_inspection=force_inspection,
  store_source=store_source,
  )
  result = loader.load(
@@ -790,6 +825,7 @@ def load_git(
  lines_collection: LinesCollection | None = None,
  modules_collection: ModulesCollection | None = None,
  allow_inspection: bool = True,
+ force_inspection: bool = False,
  find_stubs_package: bool = False,
  # TODO: Remove at some point.
  module: str | Path | None = None,
@@ -825,6 +861,7 @@ def load_git(
  lines_collection: A collection of source code lines.
  modules_collection: A collection of modules.
  allow_inspection: Whether to allow inspecting modules when visiting them is not possible.
+ force_inspection: Whether to force using dynamic analysis when loading data.
  find_stubs_package: Whether to search for stubs-only package.
  If both the package and its stubs are found, they'll be merged together.
  If only the stubs are found, they'll be used as the package itself.
@@ -856,6 +893,7 @@ def load_git(
  lines_collection=lines_collection,
  modules_collection=modules_collection,
  allow_inspection=allow_inspection,
+ force_inspection=force_inspection,
  find_stubs_package=find_stubs_package,
  # TODO: Remove at some point.
  module=module,

tests/test_loader.py

@@ -11,6 +11,7 @@
 from griffe.expressions import ExprName
 from griffe.loader import GriffeLoader
 from griffe.tests import temporary_pyfile, temporary_pypackage, temporary_visited_package
+from tests.helpers import clear_sys_modules
 
 if TYPE_CHECKING:
  from pathlib import Path
@@ -449,3 +450,17 @@ def test_side_loading_sibling_private_module(wildcard: bool, external: bool | No
  assert "foo" in package.members
  assert package["foo"].is_alias
  assert not package["foo"].resolved
+
+
+def test_forcing_inspection() -> None:
+ """Load a package with forced dynamic analysis."""
+ with temporary_pypackage("pkg", {"__init__.py": "a = 0", "mod.py": "b = 1"}) as pkg:
+ static_loader = GriffeLoader(force_inspection=False, search_paths=[pkg.tmpdir])
+ dynamic_loader = GriffeLoader(force_inspection=True, search_paths=[pkg.tmpdir])
+ static_package = static_loader.load("pkg")
+ dynamic_package = dynamic_loader.load("pkg")
+ for name in static_package.members:
+ assert name in dynamic_package.members
+ for name in static_package["mod"].members:
+ assert name in dynamic_package["mod"].members
+ clear_sys_modules("pkg")