Correct SwigPyObject_richcompare and SwigPyObject_compare undefined behaviour (#3216)

Correct SwigPyObject_richcompare and SwigPyObject_compare signatures
and avoid potential read beyond object memory.

Squashed commit of #3216 plus changes file entry and whitespace fixes.

Closes #3217

CHANGES.current

@@ -7,6 +7,10 @@ the issue number to the end of the URL: https://github.com/swig/swig/issues/
 Version 4.4.0 (in progress)
 ===========================
 
+2025-07-09: timfel
+            [Python] #3217 Fix undefined behaviour in SwigPyObject_richcompare and
+            SwigPyObject_compare.
+
 2025-06-27: wsfulton
             [Python] Raise an AttributeError instead of TypeError when there are internal
             and unexpected coding errors around member variable wrappers.

Examples/test-suite/python/python_richcompare_runme.py

@@ -83,6 +83,10 @@ if (a1 == 42) :
 if not (a1 != 42) :
     raise RuntimeError("Comparing class (with overloaded operator ==) to incompatible type, != returned False")
 
+if sys.version_info[0] >= 3:
+    if a1.__eq__(None) is not NotImplemented:
+        raise RuntimeError("Comparing to incompatible type should return NotImplemented")
+
 # Check inequalities
 #-------------------------------------------------------------------------
 

Lib/python/pyrun.swg

@@ -722,26 +722,33 @@ SwigPyObject_repr2(PyObject *v, PyObject *SWIGUNUSEDPARM(args))
 }
 
 SWIGRUNTIME int
-SwigPyObject_compare(SwigPyObject *v, SwigPyObject *w)
+SwigPyObject_compare(PyObject *v, PyObject *w)
 {
-  void *i = v->ptr;
-  void *j = w->ptr;
+  /* tp_compare is only called when both objects have the same type, so
+   * the casts are guaranteed to be ok. */
+  void *i = ((SwigPyObject *)v)->ptr;
+  void *j = ((SwigPyObject *)w)->ptr;
   return (i < j) ? -1 : ((i > j) ? 1 : 0);
 }
 
+SWIGRUNTIMEINLINE int SwigPyObject_Check(PyObject *);
+
 /* Added for Python 3.x, would it also be useful for Python 2.x? */
 SWIGRUNTIME PyObject*
-SwigPyObject_richcompare(SwigPyObject *v, SwigPyObject *w, int op)
+SwigPyObject_richcompare(PyObject *v, PyObject *w, int op)
 {
   PyObject* res = NULL;
   if (!PyErr_Occurred()) {
-    if (op != Py_EQ && op != Py_NE) {
+    /* Per https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_richcompare
+     * the first argument is guaranteed to be an instance of SwigPyObject, but the
+     * second is not, so we typecheck that one. */
+    if (op != Py_EQ && op != Py_NE || !SwigPyObject_Check(w)) {
       SWIG_Py_INCREF(Py_NotImplemented);
       return Py_NotImplemented;
     }
     res = PyBool_FromLong( (SwigPyObject_compare(v, w)==0) == (op == Py_EQ) ? 1 : 0);
   }
-  return res;  
+  return res;
 }
 
 

Source/Modules/python.cxx

@@ -4096,7 +4096,7 @@ public:
     Delete(richcompare_list);
     Printv(f, "  if (!result && !PyErr_Occurred()) {\n", NIL);
     Printv(f, "    if (SwigPyObject_Check(self) && SwigPyObject_Check(other)) {\n", NIL);
-    Printv(f, "      result = SwigPyObject_richcompare((SwigPyObject *)self, (SwigPyObject *)other, op);\n", NIL);
+    Printv(f, "      result = SwigPyObject_richcompare(self, other, op);\n", NIL);
     Printv(f, "    } else {\n", NIL);
     Printv(f, "      result = Py_NotImplemented;\n", NIL);
     Printv(f, "      SWIG_Py_INCREF(result);\n", NIL);