use .metadata distribution info when possible

When performing `install --dry-run` and PEP 658 .metadata files are available to guide the resolve, do not download the associated wheels. Rather use the distribution information directly from the .metadata files when reporting the results on the CLI and in the --report file. - describe the new --dry-run behavior - finalize linked requirements immediately after resolve - introduce is_concrete - funnel InstalledDistribution through _get_prepared_distribution() too - add test for new install --dry-run functionality (no downloading)

Author: cosmicexplorer

news/12186.bugfix.rst

@@ -0,0 +1 @@
+Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.

src/pip/_internal/commands/download.py

@@ -130,6 +130,9 @@ def run(self, options: Values, args: List[str]) -> int:
  self.trace_basic_info(finder)
 
  requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+ preparer.finalize_linked_requirements(
+ requirement_set.requirements.values(), require_dist_files=True
+ )
 
  downloaded: List[str] = []
  for req in requirement_set.requirements.values():
@@ -138,8 +141,6 @@ def run(self, options: Values, args: List[str]) -> int:
  preparer.save_linked_requirement(req)
  downloaded.append(req.name)
 
- preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-
  if downloaded:
  write_output("Successfully downloaded %s", " ".join(downloaded))
 

src/pip/_internal/commands/install.py

@@ -85,7 +85,8 @@ def add_options(self) -> None:
  help=(
  "Don't actually install anything, just print what would be. "
  "Can be used in combination with --ignore-installed "
- "to 'resolve' the requirements."
+ "to 'resolve' the requirements. If package metadata is available "
+ "or cached, --dry-run also avoids downloading the dependency at all."
  ),
  )
  self.cmd_opts.add_option(
@@ -379,6 +380,10 @@ def run(self, options: Values, args: List[str]) -> int:
  requirement_set = resolver.resolve(
  reqs, check_supported_wheels=not options.target_dir
  )
+ preparer.finalize_linked_requirements(
+ requirement_set.requirements.values(),
+ require_dist_files=not options.dry_run,
+ )
 
  if options.json_report_file:
  report = InstallationReport(requirement_set.requirements_to_install)

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,9 @@ def run(self, options: Values, args: List[str]) -> int:
  self.trace_basic_info(finder)
 
  requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+ preparer.finalize_linked_requirements(
+ requirement_set.requirements.values(), require_dist_files=True
+ )
 
  reqs_to_build: List[InstallRequirement] = []
  for req in requirement_set.requirements.values():
@@ -153,8 +156,6 @@ def run(self, options: Values, args: List[str]) -> int:
  elif should_build_for_wheel_command(req):
  reqs_to_build.append(req)
 
- preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-
  # build wheels
  build_successes, build_failures = build(
  reqs_to_build,

src/pip/_internal/distributions/__init__.py

@@ -1,4 +1,5 @@
 from pip._internal.distributions.base import AbstractDistribution
+from pip._internal.distributions.installed import InstalledDistribution
 from pip._internal.distributions.sdist import SourceDistribution
 from pip._internal.distributions.wheel import WheelDistribution
 from pip._internal.req.req_install import InstallRequirement
@@ -8,6 +9,10 @@ def make_distribution_for_install_requirement(
  install_req: InstallRequirement,
 ) -> AbstractDistribution:
  """Returns a Distribution for the given InstallRequirement"""
+ # Only pre-installed requirements will have a .satisfied_by dist.
+ if install_req.satisfied_by:
+ return InstalledDistribution(install_req)
+
  # Editable requirements will always be source distributions. They use the
  # legacy logic until we create a modern standard for them.
  if install_req.editable:

src/pip/_internal/distributions/base.py

@@ -37,11 +37,17 @@ def build_tracker_id(self) -> Optional[str]:
 
  If None, then this dist has no work to do in the build tracker, and
  ``.prepare_distribution_metadata()`` will not be called."""
- raise NotImplementedError()
+ ...
 
  @abc.abstractmethod
  def get_metadata_distribution(self) -> BaseDistribution:
- raise NotImplementedError()
+ """Generate a concrete ``BaseDistribution`` instance for this artifact.
+
+ The implementation should also cache the result with
+ ``self.req.cache_concrete_dist()`` so the distribution is available to other
+ users of the ``InstallRequirement``. This method is not called within the build
+ tracker context, so it should not identify any new setup requirements."""
+ ...
 
  @abc.abstractmethod
  def prepare_distribution_metadata(
@@ -50,4 +56,11 @@ def prepare_distribution_metadata(
  build_isolation: bool,
  check_build_deps: bool,
  ) -> None:
- raise NotImplementedError()
+ """Generate the information necessary to extract metadata from the artifact.
+
+ This method will be executed within the context of ``BuildTracker#track()``, so
+ it needs to fully identify any setup requirements so they can be added to the
+ same active set of tracked builds, while ``.get_metadata_distribution()`` takes
+ care of generating and caching the ``BaseDistribution`` to expose to the rest of
+ the resolve."""
+ ...

src/pip/_internal/distributions/installed.py

@@ -17,8 +17,10 @@ def build_tracker_id(self) -> Optional[str]:
  return None
 
  def get_metadata_distribution(self) -> BaseDistribution:
- assert self.req.satisfied_by is not None, "not actually installed"
- return self.req.satisfied_by
+ dist = self.req.satisfied_by
+ assert dist is not None, "not actually installed"
+ self.req.cache_concrete_dist(dist)
+ return dist
 
  def prepare_distribution_metadata(
  self,

src/pip/_internal/distributions/sdist.py

@@ -1,10 +1,10 @@
 import logging
-from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Iterable, Set, Tuple
 
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
-from pip._internal.metadata import BaseDistribution
+from pip._internal.metadata import BaseDistribution, get_directory_distribution
 from pip._internal.utils.subprocess import runner_with_spinner_message
 
 if TYPE_CHECKING:
@@ -21,13 +21,19 @@ class SourceDistribution(AbstractDistribution):
  """
 
  @property
- def build_tracker_id(self) -> Optional[str]:
+ def build_tracker_id(self) -> str:
  """Identify this requirement uniquely by its link."""
  assert self.req.link
  return self.req.link.url_without_fragment
 
  def get_metadata_distribution(self) -> BaseDistribution:
- return self.req.get_dist()
+ assert (
+ self.req.metadata_directory
+ ), "Set as part of .prepare_distribution_metadata()"
+ dist = get_directory_distribution(self.req.metadata_directory)
+ self.req.cache_concrete_dist(dist)
+ self.req.validate_sdist_metadata()
+ return dist
 
  def prepare_distribution_metadata(
  self,
@@ -66,7 +72,11 @@ def prepare_distribution_metadata(
  self._raise_conflicts("the backend dependencies", conflicting)
  if missing:
  self._raise_missing_reqs(missing)
- self.req.prepare_metadata()
+
+ # NB: we must still call .cache_concrete_dist() and .validate_sdist_metadata()
+ # before the InstallRequirement itself has been updated with the metadata from
+ # this directory!
+ self.req.prepare_metadata_directory()
 
  def _prepare_build_backend(self, finder: "PackageFinder") -> None:
  # Isolate in a BuildEnvironment and install the build-time

src/pip/_internal/distributions/wheel.py

@@ -31,7 +31,9 @@ def get_metadata_distribution(self) -> BaseDistribution:
  assert self.req.local_file_path, "Set as part of preparation during download"
  assert self.req.name, "Wheels are never unnamed"
  wheel = FilesystemWheel(self.req.local_file_path)
- return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+ dist = get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+ self.req.cache_concrete_dist(dist)
+ return dist
 
  def prepare_distribution_metadata(
  self,

src/pip/_internal/metadata/base.py

@@ -97,6 +97,15 @@ class RequiresEntry(NamedTuple):
 
 
 class BaseDistribution(Protocol):
+ @property
+ def is_concrete(self) -> bool:
+ """Whether the distribution really exists somewhere on disk.
+
+ If this is false, it has been synthesized from metadata, e.g. via
+ ``.from_metadata_file_contents()``, or ``.from_wheel()`` against
+ a ``MemoryWheel``."""
+ raise NotImplementedError()
+
  @classmethod
  def from_directory(cls, directory: str) -> "BaseDistribution":
  """Load the distribution from a metadata directory.
@@ -667,6 +676,10 @@ def iter_installed_distributions(
 class Wheel(Protocol):
  location: str
 
+ @property
+ def is_concrete(self) -> bool:
+ raise NotImplementedError()
+
  def as_zipfile(self) -> zipfile.ZipFile:
  raise NotImplementedError()
 
@@ -675,6 +688,10 @@ class FilesystemWheel(Wheel):
  def __init__(self, location: str) -> None:
  self.location = location
 
+ @property
+ def is_concrete(self) -> bool:
+ return True
+
  def as_zipfile(self) -> zipfile.ZipFile:
  return zipfile.ZipFile(self.location, allowZip64=True)
 
@@ -684,5 +701,9 @@ def __init__(self, location: str, stream: IO[bytes]) -> None:
  self.location = location
  self.stream = stream
 
+ @property
+ def is_concrete(self) -> bool:
+ return False
+
  def as_zipfile(self) -> zipfile.ZipFile:
  return zipfile.ZipFile(self.stream, allowZip64=True)