CLN: Use generators where list results are re-iterated over (#58296)

* Have methods returns generators instead of lists * Fix ops method, undo block * Undo internals test * Make _is_memory_usage_qualified cache_readonly too

Author: mroeschke

pandas/core/groupby/groupby.py

@@ -1286,11 +1286,10 @@ def _insert_inaxis_grouper(
  )
 
  # zip in reverse so we can always insert at loc 0
- for level, (name, lev, in_axis) in enumerate(
+ for level, (name, lev) in enumerate(
  zip(
  reversed(self._grouper.names),
- reversed(self._grouper.get_group_levels()),
- reversed([grp.in_axis for grp in self._grouper.groupings]),
+ self._grouper.get_group_levels(),
  )
  ):
  if name is None:

pandas/core/groupby/ops.py

@@ -70,6 +70,7 @@
 
 if TYPE_CHECKING:
  from collections.abc import (
+ Generator,
  Hashable,
  Iterator,
  Sequence,
@@ -857,16 +858,15 @@ def _unob_index_and_ids(
  return unob_index, unob_ids
 
  @final
- def get_group_levels(self) -> list[Index]:
+ def get_group_levels(self) -> Generator[Index, None, None]:
  # Note: only called from _insert_inaxis_grouper, which
  # is only called for BaseGrouper, never for BinGrouper
  result_index = self.result_index
  if len(self.groupings) == 1:
- return [result_index]
- return [
- result_index.get_level_values(level)
- for level in range(result_index.nlevels)
- ]
+ yield result_index
+ else:
+ for level in range(result_index.nlevels - 1, -1, -1):
+ yield result_index.get_level_values(level)
 
  # ------------------------------------------------------------
  # Aggregation functions

pandas/core/indexes/base.py

@@ -4919,6 +4919,7 @@ def _validate_fill_value(self, value):
  raise TypeError
  return value
 
+ @cache_readonly
  def _is_memory_usage_qualified(self) -> bool:
  """
  Return a boolean if we need a qualified .info display.

pandas/core/indexes/multi.py

@@ -1351,13 +1351,14 @@ def __contains__(self, key: Any) -> bool:
  def dtype(self) -> np.dtype:
  return np.dtype("O")
 
+ @cache_readonly
  def _is_memory_usage_qualified(self) -> bool:
  """return a boolean if we need a qualified .info display"""
 
  def f(level) -> bool:
  return "mixed" in level or "string" in level or "unicode" in level
 
- return any(f(level) for level in self._inferred_type_levels)
+ return any(f(level.inferred_type) for level in self.levels)
 
  # Cannot determine type of "memory_usage"
  @doc(Index.memory_usage) # type: ignore[has-type]
@@ -1659,11 +1660,6 @@ def is_monotonic_decreasing(self) -> bool:
  # monotonic decreasing if and only if reverse is monotonic increasing
  return self[::-1].is_monotonic_increasing
 
- @cache_readonly
- def _inferred_type_levels(self) -> list[str]:
- """return a list of the inferred types, one for each level"""
- return [i.inferred_type for i in self.levels]
-
  @doc(Index.duplicated)
  def duplicated(self, keep: DropKeep = "first") -> npt.NDArray[np.bool_]:
  shape = tuple(len(lev) for lev in self.levels)

pandas/core/internals/concat.py

@@ -39,7 +39,10 @@
 )
 
 if TYPE_CHECKING:
- from collections.abc import Sequence
+ from collections.abc import (
+ Generator,
+ Sequence,
+ )
 
  from pandas._typing import (
  ArrayLike,
@@ -109,12 +112,10 @@ def concatenate_managers(
  out.axes = axes
  return out
 
- concat_plan = _get_combined_plan(mgrs)
-
  blocks = []
  values: ArrayLike
 
- for placement, join_units in concat_plan:
+ for placement, join_units in _get_combined_plan(mgrs):
  unit = join_units[0]
  blk = unit.block
 
@@ -249,14 +250,12 @@ def _concat_homogeneous_fastpath(
 
 def _get_combined_plan(
  mgrs: list[BlockManager],
-) -> list[tuple[BlockPlacement, list[JoinUnit]]]:
- plan = []
-
+) -> Generator[tuple[BlockPlacement, list[JoinUnit]], None, None]:
  max_len = mgrs[0].shape[0]
 
  blknos_list = [mgr.blknos for mgr in mgrs]
  pairs = libinternals.get_concat_blkno_indexers(blknos_list)
- for ind, (blknos, bp) in enumerate(pairs):
+ for blknos, bp in pairs:
  # assert bp.is_slice_like
  # assert len(bp) > 0
 
@@ -268,9 +267,7 @@ def _get_combined_plan(
  unit = JoinUnit(nb)
  units_for_bp.append(unit)
 
- plan.append((bp, units_for_bp))
-
- return plan
+ yield bp, units_for_bp
 
 
 def _get_block_for_concat_plan(

pandas/io/formats/info.py

@@ -422,7 +422,7 @@ def size_qualifier(self) -> str:
  # categories)
  if (
  "object" in self.dtype_counts
- or self.data.index._is_memory_usage_qualified()
+ or self.data.index._is_memory_usage_qualified
  ):
  size_qualifier = "+"
  return size_qualifier

pandas/io/sql.py

@@ -177,7 +177,7 @@ def _convert_arrays_to_dataframe(
  result_arrays.append(ArrowExtensionArray(pa_array))
  arrays = result_arrays # type: ignore[assignment]
  if arrays:
- df = DataFrame(dict(zip(list(range(len(columns))), arrays)))
+ df = DataFrame(dict(zip(range(len(columns)), arrays)))
  df.columns = columns
  return df
  else: