Enable Python builtin heap types buffer interface (#3219)

For Python < 3.9 the tp_as_buffer member is set explicitly if the
interface has a bf_getbuffer slot defined. This fixes #3211.

Enabled buffer interface for non-builtin test.
This only works with Python >= 3.12, where methods __buffer__ and
__release_buffer__ were added. Unfortunately it's not practical for
these methods to reuse the slot methods (or vice versa).

Disable Py_LIMITED_API if below 3.11. The Py_buffer struct and
associated functions are not defined in earlier stable API versions.

Closes #3211

Examples/test-suite/pyabi311_bufferinterface.i

@@ -0,0 +1,124 @@
+%module pyabi311_bufferinterface
+
+%begin %{
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030b0000
+// Examples/tests-suite/python/Makefile should only run this test with Py_LIMITED_API >= 3.11
+#error testcase misconfiguration - Py_buffer is not defined in Py_LIMITED_API < 3.11
+#endif
+%}
+
+#ifdef SWIGPYTHON_BUILTIN
+%feature("python:bf_getbuffer", functype="getbufferproc") ReadOnlyData "ReadOnlyData::getbuffer";
+%feature("python:bf_releasebuffer", functype="releasebufferproc") ReadOnlyData "ReadOnlyData::releasebuffer";
+
+%feature("python:bf_getbuffer", functype="getbufferproc") ReadWriteData "ReadWriteData::getbuffer";
+%feature("python:bf_releasebuffer", functype="releasebufferproc") ReadWriteData "ReadWriteData::releasebuffer";
+#endif
+
+/* swig Python objects can only "expose" a Python buffer if the following
+ * conditions are met:
+ * 1/ SWIGPYTHON_BUILTIN is set
+ * and
+ * 2a/ Py_LIMITED_API is not set
+ * or
+ * 2b/ Py_LIMITED_API is 3.11 or greater
+ * 
+ * or
+ * 
+ * 1/ SWIGPYTHON_BUILTIN is not set
+ * and
+ * 2/ Python version is 3.12 or higher
+ */
+%inline %{
+#ifdef SWIGPYTHON_BUILTIN
+  #ifdef Py_LIMITED_API
+    #if Py_LIMITED_API+0 >= 0x030b0000
+      bool buffers_supported() { return true; }
+    #else
+      bool buffers_supported() { return false; }
+    #endif
+  #else // Py_LIMITED_API
+    bool buffers_supported() { return true; }
+  #endif // Py_LIMITED_API
+#else // SWIGPYTHON_BUILTIN
+  #if PY_VERSION_HEX >= 0x030c0000
+    bool buffers_supported() { return true; }
+  #else
+    bool buffers_supported() { return false; }
+  #endif
+#endif // SWIGPYTHON_BUILTIN
+%}
+
+%inline %{
+  class BaseClassData {
+  private:
+    char data[1024];
+  public:
+    bool released; // public so runme.py can use it as a diagnostic
+    BaseClassData() {
+      released = true;
+      strcpy(data, "This string represents a large block of memory.");
+    };
+#ifdef SWIGPYTHON_BUILTIN
+    static int getbuffer(PyObject *exporter, Py_buffer *view, int flags, bool readonly) {
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030b0000
+      goto fail;
+#endif
+      BaseClassData *self = 0;
+      if (!SWIG_IsOK(SWIG_ConvertPtr(exporter, (void**)&self, SWIGTYPE_p_BaseClassData, 0)))
+        goto fail;
+      self->released = false;
+      return PyBuffer_FillInfo(view, exporter, &self->data, sizeof(self->data), readonly ? 1 : 0, flags);
+fail:
+      PyErr_SetNone(PyExc_BufferError);
+      view->obj = NULL;
+      return -1;
+    };
+    static void releasebuffer(PyObject *exporter, Py_buffer *view) {
+      BaseClassData *self = 0;
+      if (!SWIG_IsOK(SWIG_ConvertPtr(exporter, (void**)&self, SWIGTYPE_p_BaseClassData, 0)))
+        return;
+      self->released = true;
+    };
+#else
+    PyObject* __buffer__(int flags, bool readonly) {
+      Py_buffer view;
+      if (PyBuffer_FillInfo(&view, NULL, data, sizeof(data), readonly ? 1 : 0, flags)) {
+        PyErr_SetNone(PyExc_BufferError);
+        return NULL;
+      }
+      released = false;
+      return PyMemoryView_FromBuffer(&view);
+    };
+    void __release_buffer__(PyObject* buffer) {
+      released = true;
+    };
+#endif
+  };
+
+  class ReadOnlyData: public BaseClassData {
+  public:
+#ifdef SWIGPYTHON_BUILTIN
+    static int getbuffer(PyObject *exporter, Py_buffer *view, int flags) {
+      return BaseClassData::getbuffer(exporter, view, flags, true);
+    };
+#else
+    PyObject* __buffer__(int flags) {
+      return BaseClassData::__buffer__(flags, true);
+    };
+#endif
+  };
+
+  class ReadWriteData: public BaseClassData {
+  public:
+#ifdef SWIGPYTHON_BUILTIN
+    static int getbuffer(PyObject *exporter, Py_buffer *view, int flags) {
+      return BaseClassData::getbuffer(exporter, view, flags, false);
+    };
+#else
+    PyObject* __buffer__(int flags) {
+      return BaseClassData::__buffer__(flags, false);
+    };
+#endif
+  };
+%}

Examples/test-suite/python/Makefile.in

@@ -89,6 +89,19 @@ CPP_TEST_CASES += \
 
 #	director_profile
 
+ifeq (,$(PY_ABI_VER))
+PY_ABI_311:=1
+else
+PY_ABI_VER_INTEGER:=$(shell printf "%d%02d" $(subst ., ,$(PY_ABI_VER)))
+PY_ABI_311:=$(shell test $(PY_ABI_VER_INTEGER) -ge 311 && echo "1")
+endif
+
+# Test requiring python-3.11 minimum if using Py_LIMITED_API
+ifeq (1,$(PY_ABI_311))
+CPP_TEST_CASES += pyabi311_bufferinterface
+endif
+
+
 CPP11_TEST_CASES = \
 	cpp11_hash_tables \
 	cpp11_shared_ptr_const \

Examples/test-suite/python/pyabi311_bufferinterface_runme.py

@@ -0,0 +1,40 @@
+import sys
+
+# don't bother with Python 2
+if sys.version_info < (3,0):
+    exit(0)
+
+import pyabi311_bufferinterface
+
+def check(what, expected, actual):
+    if expected != actual:
+        raise RuntimeError(
+            "Failed: ", what, " Expected: ", expected, " Actual: ", actual)
+
+# test not relevant unless certain conditions are met (builtin, limited API)
+if not pyabi311_bufferinterface.buffers_supported():
+    exit(0)
+
+data = pyabi311_bufferinterface.ReadOnlyData()
+view = memoryview(data)
+check('readonly', view.readonly, True)
+check('read data', view[:10], b"This string represents"[:10])
+check('not released', data.released, False)
+view.release()
+check('released', data.released, True)
+
+data = pyabi311_bufferinterface.ReadWriteData()
+view = memoryview(data)
+check('readonly', view.readonly, False)
+text = b'Lorem ipsum dolor sit amet'
+view[:len(text)] = b'Lorem ipsum dolor sit amet'
+check('read data', view[:len(text)], text)
+check('not released', data.released, False)
+view.release()
+check('released', data.released, True)
+
+view = memoryview(data)
+check('written data', view[:len(text)], text)
+check('not released', data.released, False)
+del view
+check('released', data.released, True)

Source/Modules/python.cxx

@@ -4522,8 +4522,12 @@ public:
     printHeapTypesSlot(f, getHeapTypesSlot(n, "feature:python:sq_inplace_repeat"), "sq_inplace_repeat", "ssizeargfunc");
 
     // buffer slots
-    printHeapTypesSlot(f, getHeapTypesSlot(n, "feature:python:bf_getbuffer"), "bf_getbuffer", "getbufferproc");
-    printHeapTypesSlot(f, getHeapTypesSlot(n, "feature:python:bf_releasebuffer"), "bf_releasebuffer", "releasebufferproc");
+    if (Getattr(n, "feature:python:bf_getbuffer")) {
+      Printv(f, "#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API+0 >= 0x030b0000\n", NIL);
+      printHeapTypesSlot(f, getHeapTypesSlot(n, "feature:python:bf_getbuffer"), "bf_getbuffer", "getbufferproc");
+      printHeapTypesSlot(f, getHeapTypesSlot(n, "feature:python:bf_releasebuffer"), "bf_releasebuffer", "releasebufferproc");
+      Printv(f, "#endif\n", NIL);
+    }
 
     Printf(f, "    { 0, NULL }\n");
     Printf(f, "  };\n");
@@ -4536,6 +4540,18 @@ public:
     Printf(f, "  };\n");
     Printv(f, "  PyObject *tuple_bases = SwigPyBuiltin_InitBases(bases);\n", NIL);
     Printf(f, "  PyTypeObject *pytype = (PyTypeObject *)PyType_FromSpecWithBases(&spec, tuple_bases);\n");
+    if (Getattr(n, "feature:python:bf_getbuffer")) {
+      Printv(f, "#if !defined(Py_LIMITED_API) && PY_VERSION_HEX < 0x03090000\n", NIL);
+      Printf(f, "  if (pytype) {\n");
+      Printf(f, "    *((PyTypeObject *)pytype)->tp_as_buffer = (PyBufferProcs){\n");
+      Printf(f, "      .bf_getbuffer = ");
+      Printv(f, getSlot(n, "feature:python:bf_getbuffer"), NIL);
+      Printf(f, ",\n      .bf_releasebuffer = ");
+      Printv(f, getSlot(n, "feature:python:bf_releasebuffer"), NIL);
+      Printf(f, "\n    };\n");
+      Printf(f, "  }\n");
+      Printv(f, "#endif\n", NIL);
+    }
     Printf(f, "  if (pytype) {\n");
     Printf(f, "    if (PyDict_Merge(pytype->tp_dict, dict, 1) == 0) {\n");
     Printv(f, "      SwigPyBuiltin_SetMetaType(pytype, type);\n", NIL);