Fix custom __str__ for enum_ (#6078)

* fix: prioritize custom enum __str__

* test: strengthen custom enum __str__ regression

* refactor: move enum __str__ handling into enum_

Keep class_::def() generic and let enum_ own the enum-specific
behavior for custom __str__ overloads. This avoids using the private
__entries attribute as a runtime sentinel for py::enum_ while preserving
the prepend behavior that lets user-defined enum __str__ methods take
precedence over the generated default.

One caveat is that this applies to normal py::enum_ API usage. Code that
intentionally upcasts an enum_ binding to class_& and then calls
class_::def("__str__", ...) will bypass this enum_ override and keep the
generic class_ behavior.

* fix: avoid enum def ambiguity on MinGW

Remove the inherited class_::def overload set from enum_ and add an explicit forwarding overload for non-string def() calls. GCC 15 on MinGW otherwise sees the duplicate dependent-base def(const char *, ...) template as ambiguous with enum_::def(const char *, ...), breaking the C++11 build.

---------

Co-authored-by: ctmd1234567 <ctmd1234567@users.noreply.github.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <rgrossekunst@nvidia.com>

include/pybind11/pybind11.h

diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
index 9cc45bd..ccd7bae 100644
--- a/include/pybind11/pybind11.h
+++ b/include/pybind11/pybind11.h
@@ -3171,7 +3171,6 @@
 public:
     using Base = class_<Type>;
     using Base::attr;
-    using Base::def;
     using Base::def_property_readonly;
     using Base::def_property_readonly_static;
     using Underlying = typename std::underlying_type<Type>::type;
@@ -3275,6 +3274,28 @@
             pos_only());
     }
 
+    template <typename Func, typename... Extra>
+    enum_ &def(const char *name_, Func &&f, const Extra &...extra) {
+        if (std::strcmp(name_, "__str__") == 0) {
+            Base::def(name_, std::forward<Func>(f), prepend{}, extra...);
+        } else {
+            Base::def(name_, std::forward<Func>(f), extra...);
+        }
+        return *this;
+    }
+
+    // Avoid using Base::def here: GCC 15/MinGW sees the duplicate dependent-base
+    // def(const char *, ...) template as ambiguous with enum_::def(const char *, ...).
+    template <typename T,
+              typename... Extra,
+              detail::enable_if_t<
+                  !std::is_convertible<typename std::decay<T>::type, const char *>::value,
+                  int> = 0>
+    enum_ &def(T &&op, const Extra &...extra) {
+        Base::def(std::forward<T>(op), extra...);
+        return *this;
+    }
+
     /// Export enumeration entries into the parent scope
     enum_ &export_values() {
         m_base.export_values();

tests/test_enum.cpp

diff --git a/tests/test_enum.cpp b/tests/test_enum.cpp
index 4ec0af7..2a7fa0f 100644
--- a/tests/test_enum.cpp
+++ b/tests/test_enum.cpp
@@ -24,6 +24,15 @@
         .value("Two", ScopedEnum::Two)
         .value("Three", ScopedEnum::Three);
 
+    // test_enum_custom_str
+    enum class CustomStrEnum { A = 1, B = 2 };
+    py::enum_<CustomStrEnum>(m, "CustomStrEnum")
+        .value("A", CustomStrEnum::A)
+        .value("B", CustomStrEnum::B)
+        .def("__str__", [](CustomStrEnum value) {
+            return "CustomStrEnum value " + std::to_string(static_cast<int>(value));
+        });
+
     m.def("test_scoped_enum", [](ScopedEnum z) {
         return "ScopedEnum::" + std::string(z == ScopedEnum::Two ? "Two" : "Three");
     });

tests/test_enum.py

diff --git a/tests/test_enum.py b/tests/test_enum.py
index 52083ca..969c30b 100644
--- a/tests/test_enum.py
+++ b/tests/test_enum.py
@@ -277,6 +277,15 @@
         assert enum_type.__str__.__doc__.startswith("__str__")
 
 
+def test_enum_custom_str_keeps_name_property():
+    assert str(m.CustomStrEnum.A) == "CustomStrEnum value 1"
+    assert str(m.CustomStrEnum.B) == "CustomStrEnum value 2"
+    assert m.CustomStrEnum.A.name == "A"
+    assert m.CustomStrEnum.A.value == 1
+    assert m.CustomStrEnum.B.name == "B"
+    assert m.CustomStrEnum.B.value == 2
+
+
 def test_generated_dunder_methods_pos_only():
     for enum_type in [m.ScopedEnum, m.UnscopedEnum]:
         for binary_op in [