Deprecate passing iterator argvalues to parametrize

Fix #13409.

Author: bluetech

changelog/13409.deprecation.rst

@@ -0,0 +1,8 @@
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects) for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>` and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is now deprecated.
+
+These iterables get exhausted after the first iteration,
+leading to tests getting unexpectedly skipped in cases such as running :func:`pytest.main()` multiple times,
+using class-level parametrize decorators,
+or collecting tests multiple times.
+
+See :ref:`parametrize-iterators` for details and suggestions.

doc/en/deprecations.rst

@@ -15,6 +15,67 @@ Below is a complete list of all pytest features which are considered deprecated.
 :class:`~pytest.PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters <warnings>`.
 
 
+.. _parametrize-iterators:
+
+Non-Collection iterables in ``@pytest.mark.parametrize``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. deprecated:: 9.1
+
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects)
+for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>`
+and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is deprecated.
+
+These iterables get exhausted after the first iteration, leads to tests getting unexpectedly skipped in cases such as:
+
+* Running :func`pytest.main()` multiple times in the same process
+* Using class-level parametrize decorators where the same mark is applied to multiple test methods
+* Collecting tests multiple times
+
+Example of problematic code:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", data_generator())
+    class Test:
+        def test_1(self, n):
+            pass
+
+        # test_2 will be skipped because data_generator() is exhausted.
+        def test_2(self, n):
+            pass
+
+You can fix it by convert generators and iterators to lists or tuples:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", list(data_generator()))
+    class Test:
+        def test_1(self, n):
+            pass
+
+        def test_2(self, n):
+            pass
+
+Note that :class:`range` objects are ``Collection`` and are not affected by this deprecation.
+
+
 .. _monkeypatch-fixup-namespace-packages:
 
 ``monkeypatch.syspath_prepend`` with legacy namespace packages

src/_pytest/compat.py

@@ -15,6 +15,7 @@
 from typing import Any
 from typing import Final
 from typing import NoReturn
+from typing import TYPE_CHECKING
 
 import py
 
@@ -311,3 +312,17 @@ def running_on_ci() -> bool:
     # Only enable CI mode if one of these env variables is defined and non-empty.
     env_vars = ["CI", "BUILD_NUMBER"]
     return any(os.environ.get(var) for var in env_vars)
+
+
+if sys.version_info >= (3, 13):
+    from warnings import deprecated as deprecated
+else:
+    if TYPE_CHECKING:
+        from typing_extensions import deprecated as deprecated
+    else:
+
+        def deprecated(msg, /, *, category=None, stacklevel=1):
+            def decorator(func):
+                return func
+
+            return decorator

src/_pytest/deprecated.py

@@ -75,6 +75,13 @@
     "See https://docs.pytest.org/en/stable/deprecations.html#monkeypatch-fixup-namespace-packages"
 )
 
+PARAMETRIZE_NON_COLLECTION_ITERABLE = UnformattedWarning(
+    PytestRemovedIn10Warning,
+    "The {param_name} parameter in parametrize uses a non-Collection iterable of type '{type_name}'.\n"
+    "Please convert to a list or tuple.\n"
+    "See https://docs.pytest.org/en/stable/deprecations.html#parametrize-iterators",
+)
+
 # You want to make some `__init__` or function "private".
 #
 #   def my_private_function(some, args):

src/_pytest/mark/structures.py

@@ -21,11 +21,13 @@
 import warnings
 
 from .._code import getfslineno
+from ..compat import deprecated
 from ..compat import NOTSET
 from ..compat import NotSetType
 from _pytest.config import Config
 from _pytest.deprecated import check_ispytest
 from _pytest.deprecated import MARKED_FIXTURE
+from _pytest.deprecated import PARAMETRIZE_NON_COLLECTION_ITERABLE
 from _pytest.outcomes import fail
 from _pytest.raises import AbstractRaises
 from _pytest.scope import _ScopeName
@@ -193,6 +195,15 @@ def _for_parametrize(
         config: Config,
         nodeid: str,
     ) -> tuple[Sequence[str], list[ParameterSet]]:
+        if not isinstance(argvalues, Collection):
+            warnings.warn(
+                PARAMETRIZE_NON_COLLECTION_ITERABLE.format(
+                    param_name="argvalues",
+                    type_name=type(argvalues).__name__,
+                ),
+                stacklevel=3,
+            )
+
         argnames, force_tuple = cls._parse_parametrize_args(argnames, argvalues)
         parameters = cls._parse_parametrize_parameters(argvalues, force_tuple)
         del argvalues
@@ -508,7 +519,25 @@ def __call__(
         ) -> MarkDecorator: ...
 
     class _ParametrizeMarkDecorator(MarkDecorator):
-        def __call__(  # type: ignore[override]
+        @overload  # type: ignore[override,no-overload-impl]
+        def __call__(
+            self,
+            argnames: str | Sequence[str],
+            argvalues: Collection[ParameterSet | Sequence[object] | object],
+            *,
+            indirect: bool | Sequence[str] = ...,
+            ids: Iterable[None | str | float | int | bool]
+            | Callable[[Any], object | None]
+            | None = ...,
+            scope: _ScopeName | None = ...,
+        ) -> MarkDecorator: ...
+
+        @overload
+        @deprecated(
+            "Passing a non-Collection iterable to the 'argvalues' parameter of @pytest.mark.parametrize is deprecated. "
+            "Convert argvalues to a list or tuple.",
+        )
+        def __call__(
             self,
             argnames: str | Sequence[str],
             argvalues: Iterable[ParameterSet | Sequence[object] | object],

src/_pytest/python.py

@@ -7,6 +7,7 @@
 from collections import Counter
 from collections import defaultdict
 from collections.abc import Callable
+from collections.abc import Collection
 from collections.abc import Generator
 from collections.abc import Iterable
 from collections.abc import Iterator
@@ -28,6 +29,7 @@
 from typing import final
 from typing import Literal
 from typing import NoReturn
+from typing import overload
 from typing import TYPE_CHECKING
 import warnings
 
@@ -41,6 +43,7 @@
 from _pytest._code.code import Traceback
 from _pytest._io.saferepr import saferepr
 from _pytest.compat import ascii_escaped
+from _pytest.compat import deprecated
 from _pytest.compat import get_default_arg_names
 from _pytest.compat import get_real_func
 from _pytest.compat import getimfunc
@@ -1209,6 +1212,34 @@ def __init__(
 
         self._params_directness: dict[str, Literal["indirect", "direct"]] = {}
 
+    @overload
+    def parametrize(
+        self,
+        argnames: str | Sequence[str],
+        argvalues: Collection[ParameterSet | Sequence[object] | object],
+        indirect: bool | Sequence[str] = False,
+        ids: Iterable[object | None] | Callable[[Any], object | None] | None = None,
+        scope: _ScopeName | None = None,
+        *,
+        _param_mark: Mark | None = None,
+    ) -> None: ...
+
+    @overload
+    @deprecated(
+        "Passing a non-Collection iterable to the 'argvalues' parameter of metafunc.parametrize is deprecated. "
+        "Convert argvalues to a list or tuple.",
+    )
+    def parametrize(
+        self,
+        argnames: str | Sequence[str],
+        argvalues: Iterable[ParameterSet | Sequence[object] | object],
+        indirect: bool | Sequence[str] = False,
+        ids: Iterable[object | None] | Callable[[Any], object | None] | None = None,
+        scope: _ScopeName | None = None,
+        *,
+        _param_mark: Mark | None = None,
+    ) -> None: ...
+
     def parametrize(
         self,
         argnames: str | Sequence[str],
@@ -1247,6 +1278,12 @@ def parametrize(
             N-tuples, where each tuple-element specifies a value for its
             respective argname.
 
+            .. versionchanged:: 9.1
+
+                Passing a non-:class:`~collections.abc.Collection` iterable
+                (such as a generator or iterator) is deprecated. See
+                :ref:`parametrize-iterators` for details.
+
         :param indirect:
             A list of arguments' names (subset of argnames) or a boolean.
             If True the list contains all names from the argnames. Each

testing/python/metafunc.py

@@ -93,7 +93,7 @@ def func(x, y):
             metafunc.parametrize("y", [5, 6])
 
         with pytest.raises(TypeError, match=r"^ids must be a callable or an iterable$"):
-            metafunc.parametrize("y", [5, 6], ids=42)  # type: ignore[arg-type]
+            metafunc.parametrize("y", [5, 6], ids=42)  # type: ignore[call-overload]
 
     def test_parametrize_error_iterator(self) -> None:
         def func(x):
@@ -134,7 +134,7 @@ def func(x):
             fail.Exception,
             match=r"parametrize\(\) call in func got an unexpected scope value 'doggy'",
         ):
-            metafunc.parametrize("x", [1], scope="doggy")  # type: ignore[arg-type]
+            metafunc.parametrize("x", [1], scope="doggy")  # type: ignore[call-overload]
 
     def test_parametrize_request_name(self, pytester: Pytester) -> None:
         """Show proper error  when 'request' is used as a parameter name in parametrize (#6183)"""
@@ -813,7 +813,7 @@ def func(x, y):
             fail.Exception,
             match="In func: expected Sequence or boolean for indirect, got dict",
         ):
-            metafunc.parametrize("x, y", [("a", "b")], indirect={})  # type: ignore[arg-type]
+            metafunc.parametrize("x, y", [("a", "b")], indirect={})  # type: ignore[call-overload]
 
     def test_parametrize_indirect_list_functional(self, pytester: Pytester) -> None:
         """
@@ -1123,6 +1123,24 @@ def test_3(self, arg, arg2):
         """
         )
 
+    def test_parametrize_iterator_deprecation(self) -> None:
+        """Test that using iterators for argvalues raises a deprecation warning."""
+
+        def func(x):
+            pass
+
+        def data_generator():
+            yield 1
+            yield 2
+
+        metafunc = self.Metafunc(func)
+
+        with pytest.warns(
+            pytest.PytestRemovedIn10Warning,
+            match=r"The argvalues parameter.*uses a non-Collection iterable",
+        ):
+            metafunc.parametrize("x", data_generator())
+
 
 class TestMetafuncFunctional:
     def test_attributes(self, pytester: Pytester) -> None:
@@ -1682,6 +1700,62 @@ def test_3(self, fixture):
             ]
         )
 
+    def test_parametrize_generator_multiple_runs(self, pytester: Pytester) -> None:
+        """Test that generators in parametrize work with multiple pytest.main() (deprecated)."""
+        pytester.makepyfile(
+            """
+            import pytest
+
+            def data_generator():
+                yield 1
+                yield 2
+
+            @pytest.mark.parametrize("bar", data_generator())
+            def test_foo(bar):
+                pass
+
+            if __name__ == '__main__':
+                args = ["-q", "--collect-only"]
+                pytest.main(args)  # First run - should work with warning
+                pytest.main(args)  # Second run - should also work with warning
+            """
+        )
+        result = pytester.runpytest("-W", "default")
+        # Should see the deprecation warnings.
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestRemovedIn10Warning: The argvalues parameter*",
+            ]
+        )
+
+    def test_parametrize_iterator_class_multiple_tests(
+        self, pytester: Pytester
+    ) -> None:
+        """Test that iterators in parametrize on a class get exhausted (deprecated)."""
+        pytester.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("n", iter(range(2)))
+            class Test:
+                def test_1(self, n):
+                    pass
+
+                def test_2(self, n):
+                    pass
+            """
+        )
+        result = pytester.runpytest("-v", "-W", "default")
+        # Iterator gets exhausted after first test, second test gets no parameters.
+        # This is deprecated.
+        result.assert_outcomes(passed=2, skipped=1)
+        # Should see the deprecation warnings.
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestRemovedIn10Warning: The argvalues parameter*",
+            ]
+        )
+
 
 class TestMetafuncFunctionalAuto:
     """Tests related to automatically find out the correct scope for

testing/typing_checks.py

@@ -8,11 +8,13 @@
 from __future__ import annotations
 
 import contextlib
+from typing import cast
 from typing import Literal
 
 from typing_extensions import assert_type
 
 import pytest
+from pytest import Metafunc
 from pytest import MonkeyPatch
 from pytest import TestReport
 
@@ -58,3 +60,16 @@ def check_raises_is_a_context_manager(val: bool) -> None:
 def check_testreport_attributes(report: TestReport) -> None:
     assert_type(report.when, Literal["setup", "call", "teardown"])
     assert_type(report.location, tuple[str, int | None, str])
+
+
+# Test @pytest.mark.parametrize iterator argvalues deprecation.
+# Will be complain about unused type ignore if doesn't work.
+@pytest.mark.parametrize("x", iter(range(10)))  # type: ignore[deprecated]
+def test_it(x: int) -> None:
+    pass
+
+
+# Test metafunc.parametrize iterator argvalues deprecation.
+# Will be complain about unused type ignore if doesn't work.
+metafunc: Metafunc = cast(Metafunc, None)
+metafunc.parametrize("x", iter(range(10)))  # type: ignore[deprecated]