From 5949fc5dc7d1e8555d350f0632ade0237c2ca52f Mon Sep 17 00:00:00 2001 From: Dag Sverre Seljebotn Date: Mon, 28 Jul 2008 20:10:53 +0200 Subject: [PATCH] Fixed and cleaned buffer acquisition (but should do more); well on the way for indirect access --- Cython/Compiler/Buffer.py | 271 +++++++++++++++++++++++------------ Cython/Compiler/ExprNodes.py | 20 ++- Cython/Compiler/Naming.py | 2 +- tests/run/bufaccess.pyx | 222 +++++++++++++++++++++++++--- 4 files changed, 400 insertions(+), 115 deletions(-) diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index c00d7478..cd15b312 100755 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -7,7 +7,7 @@ from Cython.Utils import EncodedString from Cython.Compiler.Errors import CompileError import PyrexTypes from sets import Set as set - +from textwrap import dedent class IntroduceBufferAuxiliaryVars(CythonTransform): @@ -19,12 +19,14 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): def __call__(self, node): assert isinstance(node, ModuleNode) + self.max_ndim = 0 result = super(IntroduceBufferAuxiliaryVars, self).__call__(node) if self.buffers_exists: if "endian.h" not in node.scope.include_files: node.scope.include_files.append("endian.h") use_py2_buffer_functions(node.scope) - node.scope.use_utility_code(buffer_boundsfail_error_utility_code) + use_empty_bufstruct_code(node.scope, self.max_ndim) + node.scope.use_utility_code(access_utility_code) return result @@ -48,6 +50,8 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): for entry in bufvars: name = entry.name buftype = entry.type + if buftype.ndim > self.max_ndim: + self.max_ndim = buftype.ndim # Get or make a type string checker tschecker = buffer_type_checker(buftype.dtype, scope) @@ -59,19 +63,23 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): bufinfo.used = True - def var(prefix, idx): + def var(prefix, idx, initval): cname = scope.mangle(prefix, "%d_%s" % (idx, name)) result = scope.declare_var("$%s" % cname, PyrexTypes.c_py_ssize_t_type, node.pos, cname=cname, is_cdef=True) - result.init = "0" + result.init = initval if entry.is_arg: result.used = True return result - stridevars = [var(Naming.bufstride_prefix, i) for i in range(entry.type.ndim)] - shapevars = [var(Naming.bufshape_prefix, i) for i in range(entry.type.ndim)] + stridevars = [var(Naming.bufstride_prefix, i, "0") for i in range(entry.type.ndim)] + shapevars = [var(Naming.bufshape_prefix, i, "0") for i in range(entry.type.ndim)] + suboffsetvars = [var(Naming.bufsuboffset_prefix, i, "-1") for i in range(entry.type.ndim)] entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, tschecker) + entry.buffer_aux.lookup = get_buf_lookup_full(scope, entry.type.ndim) + entry.buffer_aux.suboffsetvars = suboffsetvars + entry.buffer_aux.get_buffer_cname = tschecker scope.buffer_entries = bufvars self.scope = scope @@ -99,31 +107,24 @@ def used_buffer_aux_vars(entry): buffer_aux.buffer_info_var.used = True for s in buffer_aux.shapevars: s.used = True for s in buffer_aux.stridevars: s.used = True + for s in buffer_aux.suboffsetvars: s.used = True def put_unpack_buffer_aux_into_scope(buffer_aux, code): bufstruct = buffer_aux.buffer_info_var.cname - code.putln(" ".join(["%s = %s.strides[%d];" % - (s.cname, bufstruct, idx) - for idx, s in enumerate(buffer_aux.stridevars)])) - code.putln(" ".join(["%s = %s.shape[%d];" % - (s.cname, bufstruct, idx) - for idx, s in enumerate(buffer_aux.shapevars)])) - -def put_zero_buffer_aux_into_scope(buffer_aux, code): - # If new buffer is None, set up to access 0 - # for a "safer segfault" on access - code.putln("%s.buf = 0;" % buffer_aux.buffer_info_var.cname) - code.putln(" ".join(["%s = 0;" % s.cname - for s in buffer_aux.stridevars])) - code.putln(" ".join(["%s = 0;" % s.cname - for s in buffer_aux.shapevars])) + # __pyx_bstride_0_buf = __pyx_bstruct_buf.strides[0] and so on + + for field, vars in (("strides", buffer_aux.stridevars), + ("shape", buffer_aux.shapevars), + ("suboffsets", buffer_aux.suboffsetvars)): + code.putln(" ".join(["%s = %s.%s[%d];" % + (s.cname, bufstruct, field, idx) + for idx, s in enumerate(vars)])) def getbuffer_cond_code(obj_cname, buffer_aux, flags, ndim): bufstruct = buffer_aux.buffer_info_var.cname - checker = buffer_aux.tschecker - return "PyObject_GetBuffer(%s, &%s, %s) == -1 || %s(&%s, %d) == -1" % ( - obj_cname, bufstruct, flags, checker, bufstruct, ndim) + return "%s(%s, &%s, %s, %d) == -1" % ( + buffer_aux.get_buffer_cname, obj_cname, bufstruct, flags, ndim) def put_acquire_arg_buffer(entry, code, pos): buffer_aux = entry.buffer_aux @@ -131,10 +132,7 @@ def put_acquire_arg_buffer(entry, code, pos): bufstruct = buffer_aux.buffer_info_var.cname flags = get_flags(buffer_aux, entry.type) # Acquire any new buffer - code.put('if (%s != Py_None) ' % cname) - code.begin_block() - code.putln('%s.buf = 0;' % bufstruct) # PEP requirement - code.put(code.error_goto_if(getbuffer_cond_code(cname, + code.putln(code.error_goto_if(getbuffer_cond_code(cname, buffer_aux, flags, entry.type.ndim), @@ -142,17 +140,36 @@ def put_acquire_arg_buffer(entry, code, pos): # An exception raised in arg parsing cannot be catched, so no # need to do care about the buffer then. put_unpack_buffer_aux_into_scope(buffer_aux, code) - code.end_block() def put_release_buffer(entry, code): code.putln("if (%s != Py_None) PyObject_ReleaseBuffer(%s, &%s);" % ( entry.cname, entry.cname, entry.buffer_aux.buffer_info_var.cname)) -def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type, +def put_assign_to_buffer(lhs_cname, rhs_cname, retcode_cname, buffer_aux, buffer_type, is_initialized, pos, code): + """ + Generate code for reassigning a buffer variables. This only deals with getting + the buffer auxiliary structure and variables set up correctly, the assignment + itself and refcounting is the responsibility of the caller. + + However, the assignment operation may throw an exception so that the reassignment + never happens. + + Depending on the circumstances there are two possible outcomes: + - Old buffer released, new acquired, rhs assigned to lhs + - Old buffer released, new acquired which fails, reaqcuire old lhs buffer + (which may or may not succeed). + """ + bufstruct = buffer_aux.buffer_info_var.cname flags = get_flags(buffer_aux, buffer_type) + getbuffer = "%s(%%s, &%s, %s, %d)" % (buffer_aux.get_buffer_cname, + # note: object is filled in later + bufstruct, + flags, + buffer_type.ndim) + if is_initialized: # Release any existing buffer code.put('if (%s != Py_None) ' % lhs_cname) @@ -160,43 +177,50 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type, code.putln('PyObject_ReleaseBuffer(%s, &%s);' % ( lhs_cname, bufstruct)) code.end_block() - # Acquire any new buffer - code.put('if (%s != Py_None) ' % rhs_cname) - code.begin_block() - code.putln('%s.buf = 0;' % bufstruct) # PEP requirement - code.put('if (%s) ' % code.unlikely(getbuffer_cond_code(rhs_cname, buffer_aux, flags, buffer_type.ndim))) - code.begin_block() - # If acquisition failed, attempt to reacquire the old buffer - # before raising the exception. A failure of reacquisition - # will cause the reacquisition exception to be reported, one - # can consider working around this later. - if is_initialized: - put_zero_buffer_aux_into_scope(buffer_aux, code) - code.put('if (%s != Py_None && (%s)) ' % (rhs_cname, - getbuffer_cond_code(rhs_cname, buffer_aux, flags, buffer_type.ndim))) + # Acquire + code.putln("%s = %s;" % (retcode_cname, getbuffer % rhs_cname)) + # If acquisition failed, attempt to reacquire the old buffer + # before raising the exception. A failure of reacquisition + # will cause the reacquisition exception to be reported, one + # can consider working around this later. + code.putln('if (%s) ' % (code.unlikely("%s < 0" % retcode_cname))) + code.begin_block() + # In anticipation of a better temp system, create non-consistent C code for now + code.putln('PyObject *__pyx_type, *__pyx_value, *__pyx_tb;') + code.putln('PyErr_Fetch(&__pyx_type, &__pyx_value, &__pyx_tb);') + code.put('if (%s) ' % code.unlikely("%s == -1" % (getbuffer % lhs_cname))) code.begin_block() - put_zero_buffer_aux_into_scope(buffer_aux, code) + code.putln('Py_XDECREF(__pyx_type); Py_XDECREF(__pyx_value); Py_XDECREF(__pyx_tb);') + code.putln('PyErr_Format(PyExc_ValueError, "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!");') + code.putln('} else {') + code.putln('PyErr_Restore(__pyx_type, __pyx_value, __pyx_tb);') code.end_block() + # Unpack indices + code.end_block() + put_unpack_buffer_aux_into_scope(buffer_aux, code) + code.putln(code.error_goto_if_neg(retcode_cname, pos)) else: - # our entry had no previous value, so set to None when acquisition fails - code.putln('%s = Py_None; Py_INCREF(Py_None);' % lhs_cname) - code.putln(code.error_goto(pos)) - code.end_block() # acquisition failure - # Unpack indices - put_unpack_buffer_aux_into_scope(buffer_aux, code) - code.putln('} else {') - # If new buffer is None, set up to access 0 - # for a "safer segfault" on access - put_zero_buffer_aux_into_scope(buffer_aux, code) - code.end_block() - - # Everything is ok, assign object variable - code.putln("%s = %s;" % (lhs_cname, rhs_cname)) + # Our entry had no previous value, so set to None when acquisition fails. + # In this case, auxiliary vars should be set up right in initialization to a zero-buffer, + # so it suffices to set the buf field to NULL. + code.putln('if (%s) {' % code.unlikely("%s == -1" % (getbuffer % rhs_cname))) + code.putln('%s = Py_None; Py_INCREF(Py_None); %s.buf = NULL;' % (lhs_cname, bufstruct)) + code.putln(code.error_goto(pos)) + code.put('} else {') + # Unpack indices + put_unpack_buffer_aux_into_scope(buffer_aux, code) + code.putln('}') def put_access(entry, index_types, index_cnames, tmp_cname, pos, code): """Returns a c string which can be used to access the buffer - for reading or writing""" + for reading or writing. + + As the bounds checking can have any number of combinations of unsigned + arguments, smart optimizations etc. we insert it directly in the function + body. The lookup however is delegated to a inline function that is instantiated + once per ndim (lookup with suboffsets tend to get quite complicated). + """ bufaux = entry.buffer_aux bufstruct = bufaux.buffer_info_var.cname # Check bounds and fix negative indices @@ -237,13 +261,24 @@ def put_access(entry, index_types, index_cnames, tmp_cname, pos, code): for idx, stride in zip(index_cnames, bufaux.stridevars)]) ptrcode = "(%s.buf + %s)" % (bufstruct, offset) + ptrcode = "%s(%s.buf, %s)" % (bufaux.lookup, bufstruct, + ", ".join([", ".join([i, s.cname, o.cname]) for i, s, o in + zip(index_cnames, bufaux.stridevars, bufaux.suboffsetvars)])) valuecode = "*%s" % entry.type.buffer_ptr_type.cast_code(ptrcode) return valuecode + +def use_empty_bufstruct_code(env, max_ndim): + code = dedent(""" + Py_ssize_t __Pyx_zeros[] = {%s}; + Py_ssize_t __Pyx_minusones[] = {%s}; + """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim)) + env.use_utility_code([code, ""]) + # Utility function to set the right exception # The caller should immediately goto_error -buffer_boundsfail_error_utility_code = [ +access_utility_code = [ """\ static void __Pyx_BufferIndexError(int axis); /*proto*/ ""","""\ @@ -253,7 +288,6 @@ static void __Pyx_BufferIndexError(int axis) { } """] - # # Buffer type checking. Utility code for checking that acquired # buffers match our assumptions. We only need to check ndim and @@ -261,10 +295,18 @@ static void __Pyx_BufferIndexError(int axis) { # exporter. # buffer_check_utility_code = ["""\ +static void __Pyx_ZeroBuffer(Py_buffer* buf); /*proto*/ static const char* __Pyx_ConsumeWhitespace(const char* ts); /*proto*/ static const char* __Pyx_BufferTypestringCheckEndian(const char* ts); /*proto*/ static void __Pyx_BufferNdimError(Py_buffer* buffer, int expected_ndim); /*proto*/ """, """ +static void __Pyx_ZeroBuffer(Py_buffer* buf) { + buf->buf = NULL; + buf->strides = __Pyx_zeros; + buf->shape = __Pyx_zeros; + buf->suboffsets = __Pyx_minusones; +} + static const char* __Pyx_ConsumeWhitespace(const char* ts) { while (1) { switch (*ts) { @@ -310,6 +352,36 @@ static void __Pyx_BufferNdimError(Py_buffer* buffer, int expected_ndim) { """] + +def get_buf_lookup_full(env, nd): + """ + Generates and registers as utility a buffer lookup function for the right number + of dimensions. The function gives back a void* at the right location. + """ + name = "__Pyx_BufPtrFull_%dd" % nd + if not env.has_utility_code(name): + # _i_ndex, _s_tride, sub_o_ffset + args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)]) + proto = dedent("""\ + static INLINE void* %s(void* buf, %s); + """) % (name, args) + + func = dedent(""" + static INLINE void* %s(void* buf, %s) { + char* ptr = (char*)buf; + """) % (name, args) + "".join([dedent("""\ + ptr += s%d * i%d; + if (o%d >= 0) ptr = *((char**)ptr) + o%d; + """) % (i, i, i, i) for i in range(nd)] + ) + "\nreturn ptr;\n}" + + env.use_utility_code([proto, func], name=name) + + return name + + + + # # Utils for creating type string checkers # @@ -372,35 +444,54 @@ static const char* %s(const char* ts) { return name -def get_ts_check_simple(dtype, env): - # Check whole string for single unnamed item - name = "__Pyx_BufferTypestringCheck_simple_%s" % mangle_dtype_name(dtype) +def get_getbuffer_code(dtype, env): + """ + Generate a utility function for getting a buffer for the given dtype. + The function will: + - Call PyObject_GetBuffer + - Check that ndim matched the expected value + - Check that the format string is right + - Set suboffsets to all -1 if it is returned as NULL. + """ + + name = "__Pyx_GetBuffer_%s" % mangle_dtype_name(dtype) if not env.has_utility_code(name): - itemchecker = get_ts_check_item(dtype, env) - utilcode = [""" -static int %s(Py_buffer* buf, int e_nd); /*proto*/ -""" % name,""" -static int %(name)s(Py_buffer* buf, int e_nd) { - const char* ts = buf->format; - if (buf->ndim != e_nd) { - __Pyx_BufferNdimError(buf, e_nd); - return -1; - } - ts = __Pyx_ConsumeWhitespace(ts); - ts = __Pyx_BufferTypestringCheckEndian(ts); - if (!ts) return -1; - ts = __Pyx_ConsumeWhitespace(ts); - ts = %(itemchecker)s(ts); - if (!ts) return -1; - ts = __Pyx_ConsumeWhitespace(ts); - if (*ts != 0) { - PyErr_Format(PyExc_ValueError, - "Expected non-struct buffer data type (rejecting on '%%s')", ts); - return -1; - } - return 0; -}""" % locals()] env.use_utility_code(buffer_check_utility_code) + itemchecker = get_ts_check_item(dtype, env) + utilcode = [dedent(""" + static int %s(PyObject* obj, Py_buffer* buf, int flags, int nd); /*proto*/ + """) % name, dedent(""" + static int %(name)s(PyObject* obj, Py_buffer* buf, int flags, int nd) { + const char* ts; + if (obj == Py_None) { + __Pyx_ZeroBuffer(buf); + return 0; + } + buf->buf = NULL; + if (PyObject_GetBuffer(obj, buf, flags) == -1) goto fail; + if (buf->ndim != nd) { + __Pyx_BufferNdimError(buf, nd); + goto fail; + } + ts = buf->format; + ts = __Pyx_ConsumeWhitespace(ts); + ts = __Pyx_BufferTypestringCheckEndian(ts); + if (!ts) goto fail; + ts = __Pyx_ConsumeWhitespace(ts); + ts = %(itemchecker)s(ts); + if (!ts) goto fail; + ts = __Pyx_ConsumeWhitespace(ts); + if (*ts != 0) { + PyErr_Format(PyExc_ValueError, + "Expected non-struct buffer data type (rejecting on '%%s')", ts); + goto fail; + } + if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones; + return 0; + fail:; + __Pyx_ZeroBuffer(buf); + return -1; + }""") % locals()] env.use_utility_code(utilcode, name) return name @@ -410,7 +501,7 @@ def buffer_type_checker(dtype, env): assert False elif dtype.is_int or dtype.is_float: # This includes simple typedef-ed types - funcname = get_ts_check_simple(dtype, env) + funcname = get_getbuffer_code(dtype, env) else: assert False return funcname diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index c0cb82df..7fc0de6e 100755 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -890,7 +890,8 @@ class NameNode(AtomicExprNode): # safe than sorry. Feel free to change this. import Buffer self.new_buffer_temp = Symtab.new_temp(self.entry.type) - self.temps = [self.new_buffer_temp] + self.retcode_temp = Symtab.new_temp(PyrexTypes.c_int_type) + self.temps = [self.new_buffer_temp, self.retcode_temp] Buffer.used_buffer_aux_vars(self.entry) def analyse_rvalue_entry(self, env): @@ -1035,6 +1036,16 @@ class NameNode(AtomicExprNode): rhs.generate_disposal_code(code) else: + if self.type.is_buffer: + # Generate code for doing the buffer release/acquisition. + # This might raise an exception in which case the assignment (done + # below) will not happen. + # + # The reason this is not in a typetest-like node is because the + # variables that the acquired buffer info is stored to is allocated + # per entry and coupled with it. + self.generate_acquire_buffer(rhs, code) + if self.type.is_pyobject: rhs.make_owned_reference(code) #print "NameNode.generate_assignment_code: to", self.name ### @@ -1050,10 +1061,7 @@ class NameNode(AtomicExprNode): code.put_xdecref(self.result_code, self.ctype()) else: code.put_decref(self.result_code, self.ctype()) - if self.type.is_buffer: - self.generate_acquire_buffer(rhs, code) - else: - code.putln('%s = %s;' % (self.result_code, rhs.result_as(self.ctype()))) + code.putln('%s = %s;' % (self.result_code, rhs.result_as(self.ctype()))) if debug_disposal_code: print("NameNode.generate_assignment_code:") print("...generating post-assignment code for %s" % rhs) @@ -1066,7 +1074,7 @@ class NameNode(AtomicExprNode): code.putln('%s = %s;' % (rhstmp, rhs.result_as(self.ctype()))) import Buffer - Buffer.put_assign_to_buffer(self.result_code, rhstmp, buffer_aux, self.entry.type, + Buffer.put_assign_to_buffer(self.result_code, rhstmp, self.retcode_temp.cname, buffer_aux, self.entry.type, is_initialized=not self.skip_assignment_decref, pos=self.pos, code=code) code.putln("%s = 0;" % rhstmp) diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 30484168..90375bad 100755 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -34,7 +34,7 @@ var_prefix = pyrex_prefix + "v_" bufstruct_prefix = pyrex_prefix + "bstruct_" bufstride_prefix = pyrex_prefix + "bstride_" bufshape_prefix = pyrex_prefix + "bshape_" -bufoffset_prefix = pyrex_prefix + "boffset_" +bufsuboffset_prefix = pyrex_prefix + "boffset_" vtable_prefix = pyrex_prefix + "vtable_" vtabptr_prefix = pyrex_prefix + "vtabptr_" vtabstruct_prefix = pyrex_prefix + "vtabstruct_" diff --git a/tests/run/bufaccess.pyx b/tests/run/bufaccess.pyx index f453b7c9..5daf159e 100755 --- a/tests/run/bufaccess.pyx +++ b/tests/run/bufaccess.pyx @@ -26,6 +26,9 @@ def testcase(func): __test__[func.__name__] = setup_string + func.__doc__ return func +def testcas(a): + pass + @testcase def acquire_release(o1, o2): """ @@ -34,11 +37,16 @@ def acquire_release(o1, o2): released A acquired B released B + >>> acquire_release(None, None) + >>> acquire_release(None, B) + acquired B + released B """ cdef object[int] buf buf = o1 buf = o2 +#TODO! #@testcase def acquire_raise(o): """ @@ -60,6 +68,134 @@ def acquire_raise(o): o.printlog() raise Exception("on purpose") +@testcase +def acquire_failure1(): + """ + >>> acquire_failure1() + acquired working + 0 3 + 0 3 + released working + """ + cdef object[int] buf + buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer() + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure2(): + """ + >>> acquire_failure2() + acquired working + 0 3 + 0 3 + released working + """ + cdef object[int] buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer() + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure3(): + """ + >>> acquire_failure3() + acquired working + 0 3 + released working + acquired working + 0 3 + released working + """ + cdef object[int] buf + buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = 3 + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure4(): + """ + >>> acquire_failure4() + acquired working + 0 3 + released working + acquired working + 0 3 + released working + """ + cdef object[int] buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = 2 + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure5(): + """ + >>> acquire_failure5() + Traceback (most recent call last): + ... + ValueError: Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too! + """ + cdef object[int] buf + buf = IntMockBuffer("working", range(4)) + buf.fail = True + buf = 3 + + +@testcase +def acquire_nonbuffer1(first, second=None): + """ + >>> acquire_nonbuffer1(3) + Traceback (most recent call last): + ... + TypeError: 'int' does not have the buffer interface + >>> acquire_nonbuffer1(type) + Traceback (most recent call last): + ... + TypeError: 'type' does not have the buffer interface + >>> acquire_nonbuffer1(None, 2) + Traceback (most recent call last): + ... + TypeError: 'int' does not have the buffer interface + """ + cdef object[int] buf + buf = first + buf = second + +@testcase +def acquire_nonbuffer2(): + """ + >>> acquire_nonbuffer2() + acquired working + 0 3 + released working + acquired working + 0 3 + released working + """ + cdef object[int] buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer + assert False + except Exception: + print buf[0], buf[3] + + @testcase def as_argument(object[int] bufarg, int n): """ @@ -331,14 +467,19 @@ available_flags = ( ('WRITABLE', python_buffer.PyBUF_WRITABLE) ) +cimport stdio + cdef class MockBuffer: cdef object format - cdef char* buffer + cdef void* buffer cdef int len, itemsize, ndim cdef Py_ssize_t* strides cdef Py_ssize_t* shape + cdef Py_ssize_t* suboffsets cdef object label, log + cdef readonly object recieved_flags + cdef public object fail def __init__(self, label, data, shape=None, strides=None, format=None): self.label = label @@ -356,27 +497,78 @@ cdef class MockBuffer: cumprod *= s strides.reverse() strides = [x * self.itemsize for x in strides] + suboffsets = [-1] * len(shape) + + datashape = [len(data)] + p = data + while True: + p = p[0] + if isinstance(p, list): datashape.append(len(p)) + else: break + if len(datashape) > 1: + # indirect access + self.ndim = len(datashape) + shape = datashape + self.buffer = self.create_indirect_buffer(data, shape) + self.suboffsets = self.list_to_sizebuf(suboffsets) + else: + # strided and/or simple access + self.buffer = self.create_buffer(data) + self.ndim = len(shape) + self.suboffsets = NULL + self.format = format self.len = len(data) * self.itemsize - self.buffer = stdlib.malloc(self.len) - self.fill_buffer(data) - self.ndim = len(shape) - self.strides = stdlib.malloc(self.ndim * sizeof(Py_ssize_t)) - for i, x in enumerate(strides): - self.strides[i] = x - self.shape = stdlib.malloc(self.ndim * sizeof(Py_ssize_t)) - for i, x in enumerate(shape): - self.shape[i] = x + + self.strides = self.list_to_sizebuf(strides) + self.shape = self.list_to_sizebuf(shape) + def __dealloc__(self): stdlib.free(self.strides) stdlib.free(self.shape) - + if self.suboffsets != NULL: + stdlib.free(self.suboffsets) + # must recursively free indirect... + else: + stdlib.free(self.buffer) + + cdef void* create_buffer(self, data): + cdef char* buf = stdlib.malloc(len(data) * self.itemsize) + cdef char* it = buf + for value in data: + self.write(it, value) + it += self.itemsize + return buf + + cdef void* create_indirect_buffer(self, data, shape): + cdef void** buf + + assert shape[0] == len(data) + if len(shape) == 1: + return self.create_buffer(data) + else: + shape = shape[1:] + buf = stdlib.malloc(len(data) * sizeof(void*)) + for idx, subdata in enumerate(data): + buf[idx] = self.create_indirect_buffer(subdata, shape) + return buf + + cdef Py_ssize_t* list_to_sizebuf(self, l): + cdef Py_ssize_t* buf = stdlib.malloc(len(l) * sizeof(Py_ssize_t)) + for i, x in enumerate(l): + buf[i] = x + return buf + def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags): + if self.fail: + raise ValueError("Failing on purpose") + if buffer is NULL: print u"locking!" return self.recieved_flags = [] + cdef int value for name, value in available_flags: if (value & flags) == value: self.recieved_flags.append(name) @@ -388,7 +580,7 @@ cdef class MockBuffer: buffer.ndim = self.ndim buffer.shape = self.shape buffer.strides = self.strides - buffer.suboffsets = NULL + buffer.suboffsets = self.suboffsets buffer.itemsize = self.itemsize buffer.internal = NULL msg = "acquired %s" % self.label @@ -399,12 +591,6 @@ cdef class MockBuffer: msg = "released %s" % self.label print msg self.log += msg + "\n" - - cdef fill_buffer(self, object data): - cdef char* it = self.buffer - for value in data: - self.write(it, value) - it += self.itemsize def printlog(self): print self.log, -- 2.26.2