fix(types): type hints from future python versions (#5693)

* fix future type hints

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* remove unused var

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* remove union_helper

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* fix speelling error

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* base case for union_concat

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* add case for one descr

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* weakref and final test

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* Add acrpss_version_type_hint_checker

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* cleanup

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* remove test.pyi

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* use new unions and add fixture

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* timohl suggested cleanup

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* add missing auto

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* move operator| def

---------

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: InvincibleRMC

include/pybind11/cast.h

@@ -404,7 +404,7 @@ class type_caster<void> : public type_caster<void_type> {
     template <typename T>
     using cast_op_type = void *&;
     explicit operator void *&() { return value; }
-    static constexpr auto name = const_name("types.CapsuleType");
+    static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT);
 
 private:
     void *value = nullptr;
@@ -1361,7 +1361,7 @@ struct handle_type_name<dict> {
 };
 template <>
 struct handle_type_name<anyset> {
-    static constexpr auto name = const_name("typing.Union[set, frozenset]");
+    static constexpr auto name = const_name("set | frozenset");
 };
 template <>
 struct handle_type_name<set> {
@@ -1441,15 +1441,15 @@ struct handle_type_name<type> {
 };
 template <>
 struct handle_type_name<capsule> {
-    static constexpr auto name = const_name("types.CapsuleType");
+    static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT);
 };
 template <>
 struct handle_type_name<ellipsis> {
     static constexpr auto name = const_name("ellipsis");
 };
 template <>
 struct handle_type_name<weakref> {
-    static constexpr auto name = const_name("weakref");
+    static constexpr auto name = const_name("weakref.ReferenceType");
 };
 template <>
 struct handle_type_name<args> {

include/pybind11/detail/common.h

@@ -265,13 +265,36 @@
 #    endif
 #endif
 
+// 3.13 Compatibility
+#if 0x030D0000 <= PY_VERSION_HEX
+#    define PYBIND11_TYPE_IS_TYPE_HINT "typing.TypeIs"
+#    define PYBIND11_CAPSULE_TYPE_TYPE_HINT "types.CapsuleType"
+#else
+#    define PYBIND11_TYPE_IS_TYPE_HINT "typing_extensions.TypeIs"
+#    define PYBIND11_CAPSULE_TYPE_TYPE_HINT "typing_extensions.CapsuleType"
+#endif
+
 // 3.12 Compatibility
 #if 0x030C0000 <= PY_VERSION_HEX
 #    define PYBIND11_BUFFER_TYPE_HINT "collections.abc.Buffer"
 #else
 #    define PYBIND11_BUFFER_TYPE_HINT "typing_extensions.Buffer"
 #endif
 
+// 3.11 Compatibility
+#if 0x030B0000 <= PY_VERSION_HEX
+#    define PYBIND11_NEVER_TYPE_HINT "typing.Never"
+#else
+#    define PYBIND11_NEVER_TYPE_HINT "typing_extensions.Never"
+#endif
+
+// 3.10 Compatibility
+#if 0x030A0000 <= PY_VERSION_HEX
+#    define PYBIND11_TYPE_GUARD_TYPE_HINT "typing.TypeGuard"
+#else
+#    define PYBIND11_TYPE_GUARD_TYPE_HINT "typing_extensions.TypeGuard"
+#endif
+
 // #define PYBIND11_STR_LEGACY_PERMISSIVE
 // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject
 //             (probably surprising and never documented, but this was the

include/pybind11/detail/descr.h

@@ -157,12 +157,24 @@ constexpr descr<1, Type> _() {
 #endif // #ifndef _
 
 constexpr descr<0> concat() { return {}; }
+constexpr descr<0> union_concat() { return {}; }
 
 template <size_t N, typename... Ts>
 constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) {
     return descr;
 }
 
+template <size_t N, typename... Ts>
+constexpr descr<N, Ts...> union_concat(const descr<N, Ts...> &descr) {
+    return descr;
+}
+
+template <size_t N1, size_t N2, typename... Ts1, typename... Ts2>
+constexpr descr<N1 + N2 + 3, Ts1..., Ts2...> operator|(const descr<N1, Ts1...> &a,
+                                                       const descr<N2, Ts2...> &b) {
+    return a + const_name(" | ") + b;
+}
+
 #ifdef __cpp_fold_expressions
 template <size_t N1, size_t N2, typename... Ts1, typename... Ts2>
 constexpr descr<N1 + N2 + 2, Ts1..., Ts2...> operator,(const descr<N1, Ts1...> &a,
@@ -174,12 +186,25 @@ template <size_t N, typename... Ts, typename... Args>
 constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) {
     return (d, ..., args);
 }
+
+template <size_t N, typename... Ts, typename... Args>
+constexpr auto union_concat(const descr<N, Ts...> &d, const Args &...args) {
+    return (d | ... | args);
+}
+
 #else
 template <size_t N, typename... Ts, typename... Args>
 constexpr auto concat(const descr<N, Ts...> &d, const Args &...args)
     -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) {
     return d + const_name(", ") + concat(args...);
 }
+
+template <size_t N, typename... Ts, typename... Args>
+constexpr auto union_concat(const descr<N, Ts...> &d, const Args &...args)
+    -> decltype(std::declval<descr<N + 3, Ts...>>() + union_concat(args...)) {
+    return d + const_name(" | ") + union_concat(args...);
+}
+
 #endif
 
 template <size_t N, typename... Ts>

include/pybind11/stl.h

@@ -557,8 +557,7 @@ struct optional_caster {
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(Type,
-                         const_name("typing.Optional[") + value_conv::name + const_name("]"));
+    PYBIND11_TYPE_CASTER(Type, value_conv::name | make_caster<none>::name);
 };
 
 #if defined(PYBIND11_HAS_OPTIONAL)
@@ -642,10 +641,7 @@ struct variant_caster<V<Ts...>> {
     }
 
     using Type = V<Ts...>;
-    PYBIND11_TYPE_CASTER(Type,
-                         const_name("typing.Union[")
-                             + ::pybind11::detail::concat(make_caster<Ts>::name...)
-                             + const_name("]"));
+    PYBIND11_TYPE_CASTER(Type, ::pybind11::detail::union_concat(make_caster<Ts>::name...));
 };
 
 #if defined(PYBIND11_HAS_VARIANT)

include/pybind11/stl/filesystem.h

@@ -96,7 +96,7 @@ struct path_caster {
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, io_name("typing.Union[os.PathLike, str, bytes]", "pathlib.Path"));
+    PYBIND11_TYPE_CASTER(T, io_name("os.PathLike | str | bytes", "pathlib.Path"));
 };
 
 #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)

include/pybind11/typing.h

@@ -218,15 +218,12 @@ struct handle_type_name<typing::Type<T>> {
 
 template <typename... Types>
 struct handle_type_name<typing::Union<Types...>> {
-    static constexpr auto name = const_name("typing.Union[")
-                                 + ::pybind11::detail::concat(make_caster<Types>::name...)
-                                 + const_name("]");
+    static constexpr auto name = ::pybind11::detail::union_concat(make_caster<Types>::name...);
 };
 
 template <typename T>
 struct handle_type_name<typing::Optional<T>> {
-    static constexpr auto name
-        = const_name("typing.Optional[") + make_caster<T>::name + const_name("]");
+    static constexpr auto name = make_caster<T>::name | make_caster<none>::name;
 };
 
 template <typename T>
@@ -244,14 +241,14 @@ struct handle_type_name<typing::ClassVar<T>> {
 
 template <typename T>
 struct handle_type_name<typing::TypeGuard<T>> {
-    static constexpr auto name
-        = const_name("typing.TypeGuard[") + make_caster<T>::name + const_name("]");
+    static constexpr auto name = const_name(PYBIND11_TYPE_GUARD_TYPE_HINT) + const_name("[")
+                                 + make_caster<T>::name + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::TypeIs<T>> {
-    static constexpr auto name
-        = const_name("typing.TypeIs[") + make_caster<T>::name + const_name("]");
+    static constexpr auto name = const_name(PYBIND11_TYPE_IS_TYPE_HINT) + const_name("[")
+                                 + make_caster<T>::name + const_name("]");
 };
 
 template <>
@@ -261,7 +258,7 @@ struct handle_type_name<typing::NoReturn> {
 
 template <>
 struct handle_type_name<typing::Never> {
-    static constexpr auto name = const_name("typing.Never");
+    static constexpr auto name = const_name(PYBIND11_NEVER_TYPE_HINT);
 };
 
 #if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)

tests/conftest.py

@@ -16,6 +16,7 @@
 import sysconfig
 import textwrap
 import traceback
+from typing import Callable
 
 import pytest
 
@@ -242,3 +243,25 @@ def pytest_report_header():
         lines.append("free-threaded Python build")
 
     return lines
+
+
+@pytest.fixture
+def backport_typehints() -> Callable[[SanitizedString], SanitizedString]:
+    d = {}
+    if sys.version_info < (3, 13):
+        d["typing_extensions.TypeIs"] = "typing.TypeIs"
+        d["typing_extensions.CapsuleType"] = "types.CapsuleType"
+    if sys.version_info < (3, 12):
+        d["typing_extensions.Buffer"] = "collections.abc.Buffer"
+    if sys.version_info < (3, 11):
+        d["typing_extensions.Never"] = "typing.Never"
+    if sys.version_info < (3, 10):
+        d["typing_extensions.TypeGuard"] = "typing.TypeGuard"
+
+    def backport(sanatized_string: SanitizedString) -> SanitizedString:
+        for old, new in d.items():
+            sanatized_string.string = sanatized_string.string.replace(old, new)
+
+        return sanatized_string
+
+    return backport

tests/test_buffers.py

@@ -3,7 +3,6 @@
 import ctypes
 import io
 import struct
-import sys
 
 import pytest
 
@@ -228,12 +227,11 @@ def test_ctypes_from_buffer():
         assert not cinfo.readonly
 
 
-def test_buffer_docstring():
-    if sys.version_info >= (3, 12):
-        docstring = "get_buffer_info(arg0: collections.abc.Buffer) -> pybind11_tests.buffers.buffer_info"
-    else:
-        docstring = "get_buffer_info(arg0: typing_extensions.Buffer) -> pybind11_tests.buffers.buffer_info"
-    assert m.get_buffer_info.__doc__.strip() == docstring
+def test_buffer_docstring(doc, backport_typehints):
+    assert (
+        backport_typehints(doc(m.get_buffer_info))
+        == "get_buffer_info(arg0: collections.abc.Buffer) -> m.buffers.buffer_info"
+    )
 
 
 def test_buffer_exception():

tests/test_opaque_types.py

@@ -27,7 +27,7 @@ def test_string_list():
     assert m.print_opaque_list(cvp.stringList) == "Opaque list: [Element 1, Element 3]"
 
 
-def test_pointers(msg):
+def test_pointers(msg, backport_typehints):
     living_before = ConstructorStats.get(UserType).alive()
     assert m.get_void_ptr_value(m.return_void_ptr()) == 0x1234
     assert m.get_void_ptr_value(UserType())  # Should also work for other C++ types
@@ -37,14 +37,15 @@ def test_pointers(msg):
 
     with pytest.raises(TypeError) as excinfo:
         m.get_void_ptr_value([1, 2, 3])  # This should not work
+
     assert (
-        msg(excinfo.value)
+        backport_typehints(msg(excinfo.value))
         == """
-        get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: types.CapsuleType) -> int
+            get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
+                1. (arg0: types.CapsuleType) -> int
 
-        Invoked with: [1, 2, 3]
-    """
+            Invoked with: [1, 2, 3]
+        """
     )
 
     assert m.return_null_str() is None

tests/test_pytypes.cpp

@@ -155,7 +155,7 @@ namespace detail {
 
 template <>
 struct type_caster<RealNumber> {
-    PYBIND11_TYPE_CASTER(RealNumber, io_name("typing.Union[float, int]", "float"));
+    PYBIND11_TYPE_CASTER(RealNumber, io_name("float | int", "float"));
 
     static handle cast(const RealNumber &number, return_value_policy, handle) {
         return py::float_(number.value).release();