Ctypes from_buffer with memoryviews in Python 2.7 and Python 3.4

Question

I am trying to pass data from a memoryview to a ctypes array, which works fine in Python 3.4 but not in Python 2.7.

When I run

from ctypes import c_byte
data = memoryview(b'012')
array = c_byte * 3
array.from_buffer_copy(data)

I get <__main__.c_byte_Array_3 at 0x7f3022cb8730> in Python 3.4, but in Python 2.7.6 I get the following error:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: expected a readable buffer object

What is the reason for this error and how I can make this work in both cases?

I know I can convert the data to bytes using

array.from_buffer_copy(data.tobytes())

but I think that makes one additional copy of the data and is not elegant, so I am looking for a better solution (any comments on whether the tobytes method is efficient or not would be welcome as well).

Bucephalus

2.x ctypes is calling PyObject_AsReadBuffer, which looks for the old buffer protocol slot bf_getreadbuffer. The memoryview type doesn't define this; it uses the new bf_getbuffer interface. – Zebedee

Answer 1

Here's a class that will let you create ctypes arrays using the buffer interface exported by Python 2 memoryview objects.

from ctypes import *

pyapi = PyDLL("PythonAPI", handle=pythonapi._handle)

PyBUF_SIMPLE   = 0
PyBUF_WRITABLE = 0x0001
PyBUF_FORMAT   = 0x0004
PyBUF_ND       = 0x0008
PyBUF_STRIDES  = 0x0010 | PyBUF_ND

PyBUF_C_CONTIGUOUS   = 0x0020 | PyBUF_STRIDES
PyBUF_F_CONTIGUOUS   = 0x0040 | PyBUF_STRIDES
PyBUF_ANY_CONTIGUOUS = 0x0080 | PyBUF_STRIDES
PyBUF_INDIRECT       = 0x0100 | PyBUF_STRIDES

PyBUF_CONTIG_RO  = PyBUF_ND
PyBUF_CONTIG     = PyBUF_ND | PyBUF_WRITABLE

PyBUF_STRIDED_RO = PyBUF_STRIDES
PyBUF_STRIDED    = PyBUF_STRIDES | PyBUF_WRITABLE

PyBUF_RECORDS_RO = PyBUF_STRIDES | PyBUF_FORMAT
PyBUF_RECORDS    = PyBUF_STRIDES | PyBUF_FORMAT | PyBUF_WRITABLE

PyBUF_FULL_RO = PyBUF_INDIRECT | PyBUF_FORMAT
PyBUF_FULL    = PyBUF_INDIRECT | PyBUF_FORMAT | PyBUF_WRITABLE

Py_ssize_t = c_ssize_t
Py_ssize_t_p = POINTER(Py_ssize_t)

class pybuffer(Structure):
    """Python 3 Buffer Interface"""
    _fields_ = (('buf', c_void_p),
                ('obj', c_void_p), # owned reference
                ('len', Py_ssize_t),
                # itemsize is Py_ssize_t so it can be pointed to
                # by strides in the simple case.
                ('itemsize', Py_ssize_t),
                ('readonly', c_int),
                ('ndim', c_int),
                ('format', c_char_p),
                ('shape', Py_ssize_t_p),
                ('strides', Py_ssize_t_p),
                ('suboffsets', Py_ssize_t_p),
                # static store for shape and strides of
                # mono-dimensional buffers.
                ('smalltable', Py_ssize_t * 2),
                ('internal', c_void_p))

    def get_buffer(self, obj=None, flags=PyBUF_SIMPLE):
        self.release_buffer()
        Structure.__init__(self)
        if obj is not None:
            pyapi.PyObject_GetBuffer(obj, byref(self), flags)

    def make_release_buffer():
        import ctypes
        PyBuffer_Release = pyapi.PyBuffer_Release
        memset = ctypes.memset
        byref = ctypes.byref
        sizeof = ctypes.sizeof
        def release_buffer(self):
            if self.obj:
                PyBuffer_Release(byref(self))
                memset(byref(self), 0, sizeof(self))
        return release_buffer

    __init__ = get_buffer
    __del__ = release_buffer = make_release_buffer()
    del make_release_buffer        

    @property
    def as_ctypes(self):
        if self.obj and self.buf:
            arr = (c_char * self.len).from_address(self.buf)
            if self.readonly:
                arr = type(arr).from_buffer_copy(arr)
            else:
                obj = py_object.from_buffer(c_void_p(self.obj)).value
                arr._obj = obj
            return arr


pyapi.PyObject_GetBuffer.argtypes = (py_object,          # obj
                                     POINTER(pybuffer),  # view
                                     c_int)              # flags
pyapi.PyBuffer_Release.argtypes = POINTER(pybuffer),     # view

__all__ = [n for n in list(globals()) if n.startswith('PyBUF')]
__all__.append('pybuffer')

Examples:

>>> data = memoryview(b'012')
>>> buf = pybuffer(data)
>>> buf.readonly
1
>>> array = buf.as_ctypes
>>> array[0] = '9'
>>> data[0]
'0'
>>> data = memoryview(bytearray(b'012'))
>>> buf = pybuffer(data)
>>> buf.readonly
0
>>> array = buf.as_ctypes
>>> array[0] = '9'
>>> data[0]
'9'

Zebedee