I have written an extension module that uses C++ function pointers to store sequences of function calls. I want to 'run' these call sequences in separate processes using python's multiprocessing
module (there's no shared state, so no synchronization issues).
I need to know if function pointers (not data pointers) remain valid after multiprocessing
does it's fork()
.
C++ module:
#include <list>
#include <boost/assert.hpp>
#include <boost/python.hpp>
#include <boost/python/stl_iterator.hpp>
#include <boost/foreach.hpp>
/*
* Some functions to be called
*/
double funcA(double d) { return d; }
double funcB(double d) { return d + 3.14; }
double funcC(double d) { return d - 42.0; }
/*
* My container of function pointers (picklable to allow use with multiprocessing)
*/
typedef double(*func_ptr_t)(double);
struct CallSequence {
CallSequence() {
_seq.push_back(funcA);
_seq.push_back(funcB);
_seq.push_back(funcC);
}
std::list<func_ptr_t> _seq;
};
template <typename cast_type>
struct CallSequence_picklesuite : boost::python::pickle_suite {
BOOST_STATIC_ASSERT_MSG(sizeof(cast_type) == sizeof(func_ptr_t), CANNOT_CAST_POINTER_TO_REQUESTED_TYPE);
static boost::python::list getstate(const CallSequence& cs) {
boost::python::list ret;
BOOST_FOREACH(func_ptr_t p, cs._seq)
ret.append(reinterpret_cast<cast_type>(p));
return ret;
}
static void setstate(CallSequence& cs, boost::python::list l) {
std::list<func_ptr_t> new_list;
boost::python::stl_input_iterator<cast_type> begin(l), end;
for(; begin != end; begin++)
new_list.push_back(reinterpret_cast<func_ptr_t>(*begin));
cs._seq.swap(new_list);
}
};
/*
* Run the call sequence
*/
double runner(const CallSequence& cs) {
double ret = 0;
BOOST_FOREACH(const func_ptr_t& p, cs._seq)
ret += p(2.18);
return ret;
}
BOOST_PYTHON_MODULE(my_extension) {
using namespace ::boost::python;
class_<CallSequence>("CallSequence")
.def_pickle(CallSequence_picklesuite<unsigned int>());
def("runner", runner);
}
Compiled with:
$ g++ question1.cpp -lboost_python -I /usr/include/python2.7 -shared -o my_extension.so
Python code invoking it across multiple processes:
#!/usr/bin/python
from multiprocessing import Pool
import my_extension
def runner(sequence):
return my_extension.runner(sequence)
def main():
l = [my_extension.CallSequence() for _ in range(200)]
pool = Pool(processes=4)
print pool.map(runner, l)
if __name__ == '__main__':
main()
The output is as expected. I want to know if I'm just 'getting lucky' or if I can reliably expect function pointers to remain valid after a fork()
.
fork
copies the address space. My answer would have been correct for two completely unrelated processes, which I thought is whatfork
does, but of course I was wrong. +1 – Susurrus