Allow different test output for different Python versions (#10382)

Author: cdce8p

doc/development_guide/contributor_guide/tests/writing_test.rst

@@ -66,6 +66,32 @@ test runner. The following options are currently supported:
 - "except_implementations": List of python implementations on which the test should not run
 - "exclude_platforms": List of operating systems on which the test should not run
 
+**Different output for different Python versions**
+
+Sometimes the linting result can change between Python releases. In these cases errors can be marked as conditional.
+Supported operators are ``<``, ``<=``, ``>`` and ``>=``.
+
+.. code-block:: python
+
+ def some_func() -> X: # <3.14:[undefined-variable]
+ ...
+
+ # It can also be combined with offsets
+ # +1:<3.14:[undefined-variable]
+ def some_other_func() -> X:
+ ...
+
+ class X: ...
+
+Since the output messages are different, it is necessary to add two separate files for it.
+First ``<test-file-name>.314.txt``, this will include the output messages for ``>=3.14``, i.e. should be empty here.
+Second ``<test-file-name>.txt``, this will be the default for all other Python versions.
+
+.. note::
+
+ This does only work if the code itself is parsable in all tested Python versions.
+ For new syntax, use ``min_pyver`` / ``max_pyver`` instead.
+
 **Functional test file locations**
 
 For existing checkers, new test cases should preferably be appended to the existing test file.

doc/whatsnew/fragments/10382.internal

@@ -0,0 +1,3 @@
+Modified test framework to allow for different test output for different Python versions.
+
+Refs #10382

pylint/testutils/functional/test_file.py

@@ -5,9 +5,13 @@
 from __future__ import annotations
 
 import configparser
+import sys
 from collections.abc import Callable
-from os.path import basename, exists, join
-from typing import TypedDict
+from os.path import basename, exists, join, split
+from pathlib import Path
+from typing import Final, TypedDict
+
+_CURRENT_VERSION: Final = sys.version_info[:2]
 
 
 def parse_python_version(ver_str: str) -> tuple[int, ...]:
@@ -99,7 +103,20 @@ def module(self) -> str:
 
  @property
  def expected_output(self) -> str:
- return self._file_type(".txt", check_exists=False)
+ files = [
+ p.stem
+ for p in Path(self._directory).glob(f"{split(self.base)[-1]}.[0-9]*.txt")
+ ]
+ output_options = [
+ (int(version[0]), int(version[1:]))
+ for s in files
+ if (version := s.rpartition(".")[2]).isalnum()
+ ]
+ for opt in sorted(output_options, reverse=True):
+ if _CURRENT_VERSION >= opt:
+ str_opt = "".join([str(s) for s in opt])
+ return join(self._directory, f"{self.base}.{str_opt}.txt")
+ return join(self._directory, self.base + ".txt")
 
  @property
  def source(self) -> str:

tests/functional/ext/typing/unnecessary_default_type_args_py313.txt renamed to tests/functional/ext/typing/unnecessary_default_type_args.313.txt

@@ -3,10 +3,10 @@
 import typing as t
 
 a1: t.Generator[int, str, str]
-a2: t.Generator[int, None, None]
+a2: t.Generator[int, None, None] # >=3.13:[unnecessary-default-type-args]
 a3: t.Generator[int]
 b1: t.AsyncGenerator[int, str]
-b2: t.AsyncGenerator[int, None]
+b2: t.AsyncGenerator[int, None] # >=3.13:[unnecessary-default-type-args]
 b3: t.AsyncGenerator[int]
 
 c1: ca.Generator[int, str, str]

tests/functional/ext/typing/unnecessary_default_type_args.py

@@ -1,3 +1,2 @@
 [main]
-py-version=3.10
 load-plugins=pylint.extensions.typing

tests/functional/ext/typing/unnecessary_default_type_args.rc

@@ -4,8 +4,14 @@
 
 """Tests for the functional test framework."""
 
+import contextlib
+import os
+import os.path
+import shutil
+import tempfile
+from collections.abc import Iterator
 from pathlib import Path
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 from _pytest.outcomes import Skipped
@@ -20,6 +26,26 @@
 DATA_DIRECTORY = HERE / "data"
 
 
+@contextlib.contextmanager
+def tempdir() -> Iterator[str]:
+ """Create a temp directory and change the current location to it.
+
+ This is supposed to be used with a *with* statement.
+ """
+ tmp = tempfile.mkdtemp()
+
+ # Get real path of tempfile, otherwise test fail on mac os x
+ current_dir = os.getcwd()
+ os.chdir(tmp)
+ abs_tmp = os.path.abspath(".")
+
+ try:
+ yield abs_tmp
+ finally:
+ os.chdir(current_dir)
+ shutil.rmtree(abs_tmp)
+
+
 @pytest.fixture(name="pytest_config")
 def pytest_config_fixture() -> MagicMock:
  def _mock_getoption(option: str) -> bool:
@@ -69,6 +95,37 @@ def test_get_functional_test_files_from_crowded_directory() -> None:
  assert "max_overflow" not in str(exc_info.value)
 
 
+@pytest.mark.parametrize(
+ ["files", "output_file_name"],
+ [
+ ([], "file.txt"),
+ (["file.txt"], "file.txt"),
+ (["file.314.txt"], "file.txt"), # don't match 3.14
+ (["file.42.txt"], "file.txt"), # don't match 4.2
+ (["file.32.txt", "file.txt"], "file.32.txt"),
+ (["file.312.txt", "file.txt"], "file.312.txt"),
+ (["file.313.txt", "file.txt"], "file.313.txt"),
+ (["file.310.txt", "file.313.txt", "file.312.txt", "file.txt"], "file.313.txt"),
+ # don't match other test file names accidentally
+ ([".file.313.txt"], "file.txt"),
+ (["file_other.313.txt"], "file.txt"),
+ (["other_file.313.txt"], "file.txt"),
+ ],
+)
+def test_expected_output_file_matching(files: list[str], output_file_name: str) -> None:
+ """Test output file matching. Pin current Python version to 3.13."""
+ with tempdir():
+ for file in files:
+ with open(file, "w", encoding="utf-8"):
+ ...
+ test_file = FunctionalTestFile(".", "file.py")
+ with patch(
+ "pylint.testutils.functional.test_file._CURRENT_VERSION",
+ new=(3, 13),
+ ):
+ assert test_file.expected_output == f".{os.path.sep}{output_file_name}"
+
+
 def test_minimal_messages_config_enabled(pytest_config: MagicMock) -> None:
  """Test that all messages not targeted in the functional test are disabled
  when running with --minimal-messages-config.