gh-127065: Make methodcaller thread-safe and re-entrant (v2) by eendebakpt · Pull Request #127746 · python/cpython

python

@@ -1595,78 +1595,75 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; PyObject *xargs; // reference to arguments passed in constructor PyObject *args; PyObject *kwds; PyObject **vectorcall_args; /* Borrowed references */ PyObject *vectorcall_args; PyObject *vectorcall_kwnames; vectorcallfunc vectorcall; } methodcallerobject;
#ifndef Py_GIL_DISABLED static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) { PyObject* args = mc->xargs; PyObject* kwds = mc->kwds;
Py_ssize_t nargs = PyTuple_GET_SIZE(args); assert(nargs > 0); mc->vectorcall_args = PyMem_Calloc( nargs + (kwds ? PyDict_Size(kwds) : 0), sizeof(PyObject*)); if (!mc->vectorcall_args) { PyErr_NoMemory(); return -1; } /* The first item of vectorcall_args will be filled with obj later */ if (nargs > 1) { memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), nargs * sizeof(PyObject*)); } if (kwds) { const Py_ssize_t nkwds = PyDict_Size(kwds);
mc->vectorcall_kwnames = PyTuple_New(nkwds); if (!mc->vectorcall_kwnames) { return -1; } Py_ssize_t i = 0, ppos = 0; PyObject* key, * value; while (PyDict_Next(kwds, &ppos, &key, &value)) { PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); mc->vectorcall_args[nargs + i] = value; // borrowed reference ++i; } } else { mc->vectorcall_kwnames = NULL; } return 1; }
#define _METHODCALLER_MAX_ARGS 8
static PyObject * methodcaller_vectorcall( methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) methodcaller_vectorcall(methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) { if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } if (mc->vectorcall_args == NULL) { if (_methodcaller_initialize_vectorcall(mc) < 0) { return NULL; } } assert(mc->vectorcall_args != NULL);
PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; tmp_args[0] = args[0]; assert(1 + PyTuple_GET_SIZE(mc->vectorcall_args) <= _METHODCALLER_MAX_ARGS); memcpy(tmp_args + 1, _PyTuple_ITEMS(mc->vectorcall_args), sizeof(PyObject *) * PyTuple_GET_SIZE(mc->vectorcall_args));
assert(mc->vectorcall_args != 0); mc->vectorcall_args[0] = args[0]; return PyObject_VectorcallMethod( mc->name, mc->vectorcall_args, (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, return PyObject_VectorcallMethod(mc->name, tmp_args, (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); } #endif
static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) { PyObject* args = mc->args; PyObject* kwds = mc->kwds;
if (kwds && PyDict_Size(kwds)) { PyObject *values = PyDict_Values(kwds); if (!values) { return -1; } PyObject *values_tuple = PySequence_Tuple(values); Py_DECREF(values); if (!values_tuple) { return -1; } if (PyTuple_GET_SIZE(args)) { mc->vectorcall_args = PySequence_Concat(args, values_tuple); Py_DECREF(values_tuple); if (mc->vectorcall_args == NULL) { return -1; } } else { mc->vectorcall_args = values_tuple; } mc->vectorcall_kwnames = PySequence_Tuple(kwds); if (!mc->vectorcall_kwnames) { return -1; } } else { mc->vectorcall_args = Py_NewRef(args); mc->vectorcall_kwnames = NULL; }
mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; return 0; }
/* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * @@ -1694,25 +1691,30 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (mc == NULL) { return NULL; } mc->vectorcall = NULL; mc->vectorcall_args = NULL; mc->vectorcall_kwnames = NULL; mc->kwds = Py_XNewRef(kwds);
Py_INCREF(name); PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternMortal(interp, &name); mc->name = name;
mc->xargs = Py_XNewRef(args); // allows us to use borrowed references mc->kwds = Py_XNewRef(kwds); mc->vectorcall_args = 0;
mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); if (mc->args == NULL) { Py_DECREF(mc); return NULL; }
#ifdef Py_GIL_DISABLED // gh-127065: The current implementation of methodcaller_vectorcall // is not thread-safe because it modifies the `vectorcall_args` array, // which is shared across calls. mc->vectorcall = NULL; #else mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; #endif Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + (kwds ? PyDict_Size(kwds) : 0); if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { if (_methodcaller_initialize_vectorcall(mc) < 0) { Py_DECREF(mc); return NULL; } }
PyObject_GC_Track(mc); return (PyObject *)mc; @@ -1722,13 +1724,10 @@ static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); Py_CLEAR(mc->xargs); Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); if (mc->vectorcall_args != NULL) { PyMem_Free(mc->vectorcall_args); mc->vectorcall_args = 0; Py_CLEAR(mc->vectorcall_kwnames); } Py_CLEAR(mc->vectorcall_args); Py_CLEAR(mc->vectorcall_kwnames); }
static void @@ -1745,8 +1744,10 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); Py_VISIT(mc->xargs); Py_VISIT(mc->args); Py_VISIT(mc->kwds); Py_VISIT(mc->vectorcall_args); Py_VISIT(mc->vectorcall_kwnames); Py_VISIT(Py_TYPE(mc)); return 0; } @@ -1765,15 +1766,7 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) if (method == NULL) return NULL;

PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); if (cargs == NULL) { Py_DECREF(method); return NULL; }
result = PyObject_Call(method, cargs, mc->kwds); Py_DECREF(cargs); result = PyObject_Call(method, mc->args, mc->kwds); Py_DECREF(method); return result; } @@ -1791,7 +1784,7 @@ methodcaller_repr(methodcallerobject *mc) }
numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; numposargs = PyTuple_GET_SIZE(mc->args); numtotalargs = numposargs + numkwdargs;
if (numtotalargs == 0) { @@ -1807,7 +1800,7 @@ methodcaller_repr(methodcallerobject *mc) }
for (i = 0; i < numposargs; ++i) { PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1859,14 +1852,14 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); PyObject *newargs = PyTuple_New(newarg_size); Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); PyObject *newargs = PyTuple_New(1 + callargcount); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); for (i = 1; i < newarg_size; ++i) { PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); for (i = 0; i < callargcount; ++i) { PyObject *arg = PyTuple_GET_ITEM(mc->args, i); PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1884,12 +1877,7 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds);
Py_DECREF(partial); PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); if (!args) { Py_DECREF(constructor); return NULL; } return Py_BuildValue("NO", constructor, args); return Py_BuildValue("NO", constructor, mc->args); } }