Menu
▾
▴
[simuPOP-devel] SF.net SVN: simupop:[4983] trunk/src
|
From: <bpe...@us...> - 2015-08-29 16:10:35
|
Revision: 4983
http://sourceforge.net/p/simupop/code/4983
Author: bpeng2000
Date: 2015-08-29 16:10:32 +0000 (Sat, 29 Aug 2015)
Log Message:
-----------
Use pickle to load and save population variables
Modified Paths:
--------------
trunk/src/customizedTypes.c
trunk/src/population.cpp
trunk/src/population.h
trunk/src/utility.cpp
trunk/src/utility.h
Modified: trunk/src/customizedTypes.c
===================================================================
--- trunk/src/customizedTypes.c 2015-08-25 03:32:09 UTC (rev 4982)
+++ trunk/src/customizedTypes.c 2015-08-29 16:10:32 UTC (rev 4983)
@@ -806,7 +806,10 @@
}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
#else // for Python 3
+
/* Array object implementation */
typedef struct arrayobject_template<GenoIterator> arrayobject;
@@ -1364,7 +1367,7 @@
return NULL;
}
result = PyTuple_Pack(4, Py_TYPE(dd),
- Py_None, Py_None, iter);
+ PyTuple_New(0), Py_None, iter);
Py_DECREF(iter);
Py_DECREF(items);
return result;
@@ -1442,7 +1445,7 @@
PyTypeObject defdict_type = {
PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0)
- "simuPOP.defaultdict", /* tp_name */
+ "defdict", /* tp_name */
sizeof(defdictobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
Modified: trunk/src/population.cpp
===================================================================
--- trunk/src/population.cpp 2015-08-25 03:32:09 UTC (rev 4982)
+++ trunk/src/population.cpp 2015-08-29 16:10:32 UTC (rev 4983)
@@ -3392,7 +3392,7 @@
}
-void Population::save(boost::archive::text_oarchive & ar, const unsigned int) const
+void Population::save(boost::archive::text_oarchive & ar, const unsigned int version) const
{
// deep adjustment: everyone in order
const_cast<Population *>(this)->syncIndPointers();
@@ -3631,7 +3631,8 @@
// save shared variables as string.
// note that many format are not supported.
DBG_DO(DBG_POPULATION, cerr << "Handling shared variables" << endl);
- string vars = varsAsString();
+
+ string vars = varsAsString(version >= 3);
ar & vars;
}
@@ -3653,7 +3654,7 @@
DBG_DO(DBG_POPULATION, cerr << "Handling genotype" << endl);
// newer version unfied importer
- if (version == 2) {
+ if (version >= 2) {
// a newer version
size_t size;
ar & size;
@@ -3873,7 +3874,7 @@
ar & pd.m_subPopSize;
ar & pd.m_subPopNames;
- if (version == 2) {
+ if (version >= 2) {
// a newer version
size_t size;
ar & size;
@@ -4063,8 +4064,8 @@
DBG_DO(DBG_POPULATION, cerr << "Handling shared variables" << endl);
string vars;
ar & vars;
- varsFromString(vars);
+ varsFromString(vars, version >= 3);
setIndOrdered(true);
DBG_WARNIF(max_allele > ModuleMaxAllele, (boost::format("Warning: the maximum allele of the loaded population is %1%"
" which is larger than the maximum allowed allele of this module. "
Modified: trunk/src/population.h
===================================================================
--- trunk/src/population.h 2015-08-25 03:32:09 UTC (rev 4982)
+++ trunk/src/population.h 2015-08-29 16:10:32 UTC (rev 4983)
@@ -1596,16 +1596,22 @@
/// CPPONLY
- string varsAsString() const
+ string varsAsString(bool use_pickle=false) const
{
- return m_vars.asString();
+ if (use_pickle)
+ return m_vars.to_pickle();
+ else
+ return m_vars.asString();
}
/// CPPONLY
- void varsFromString(const string & vars)
+ void varsFromString(const string & vars, bool use_pickle=false)
{
- return m_vars.fromString(vars);
+ if (use_pickle)
+ return m_vars.from_pickle(vars);
+ else
+ return m_vars.fromString(vars);
}
@@ -1768,7 +1774,8 @@
// version 0: base (reset for version 1.0)
// version 1: with lineage information for lineage-aware modules
// version 2: for memory-efficient save/load
-BOOST_CLASS_VERSION(simuPOP::Population, 2)
+// version 3: use pickle to save load population variables
+BOOST_CLASS_VERSION(simuPOP::Population, 3)
# endif
#endif
#endif
Modified: trunk/src/utility.cpp
===================================================================
--- trunk/src/utility.cpp 2015-08-25 03:32:09 UTC (rev 4982)
+++ trunk/src/utility.cpp 2015-08-29 16:10:32 UTC (rev 4983)
@@ -2396,6 +2396,35 @@
}
+string SharedVariables::to_pickle() const
+{
+ PyObject * pickle = PyImport_ImportModule("pickle");
+ if (! pickle)
+ throw RuntimeError("Failed to import module pickle to serialize population variables.");
+
+ // here we use version 2 because this is the latest version that supported by
+ // both python 2 and python 3. This is a binary format so both point and size
+ // are needed.
+ PyObject * pres = PyObject_CallMethod(pickle, "dumps", "(Oi)", m_dict, 2);
+ if (pres == NULL) {
+ PyErr_Print();
+ PyErr_Clear();
+ throw RuntimeError("Failed to call pickle.dumps to save population variables.");
+ }
+ Py_ssize_t sz = 0;
+ char * buf = NULL;
+#if PY_VERSION_HEX >= 0x03000000
+ PyBytes_AsStringAndSize(pres, &buf, &sz);
+#else
+ PyString_AsStringAndSize(pres, &buf, &sz);
+#endif
+ Py_DECREF(pres);
+ Py_DECREF(pickle);
+
+ return string(buf, sz);
+}
+
+
string SharedVariables::asString() const
{
// go through each variable and save
@@ -2424,7 +2453,36 @@
m_dict = obj;
}
+void SharedVariables::from_pickle(const string & vars)
+{
+ PyObject * pickle = PyImport_ImportModule("pickle");
+ if (! pickle)
+ throw RuntimeError("Failed to import module pickle to serialize population variables.");
+ // use protocol 1 for compatibility
+ string res;
+#if PY_VERSION_HEX >= 0x03000000
+ PyObject * args = PyBytes_FromStringAndSize(vars.c_str(), vars.size());
+#else
+ PyObject * args = PyString_FromStringAndSize(vars.c_str(), vars.size());
+#endif
+ // remove m_dict
+ if (m_ownVars) {
+ PyDict_Clear(m_dict);
+ Py_XDECREF(m_dict);
+ }
+ m_ownVars = true;
+ m_dict = PyObject_CallMethod(pickle, "loads", "(O)", args);
+ if (m_dict == NULL) {
+ PyErr_Print();
+ PyErr_Clear();
+ throw RuntimeError("Failed to call pickle.loads to load population variables.");
+ }
+ Py_DECREF(args);
+
+ Py_DECREF(pickle);
+}
+
// simuVars will hold replicate specific Shared Variables.
// global dictionary
Modified: trunk/src/utility.h
===================================================================
--- trunk/src/utility.h 2015-08-25 03:32:09 UTC (rev 4982)
+++ trunk/src/utility.h 2015-08-29 16:10:32 UTC (rev 4983)
@@ -1264,8 +1264,15 @@
/// CPPONLY, save m_dist as string
string asString() const;
+ /// CPPONLY
void fromString(const string & vars);
+ /// CPPONLY, save m_dist to pickle
+ string to_pickle() const;
+
+ /// CPPONLY
+ void from_pickle(const string & vars);
+
private:
/// the list
PyObject * m_dict;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|