Bitton Tenessi
2015-01-17 19:53:32 UTC
Hi,
Has anyone tried to deploy theano on Heroku? Numpy and Scipy are usually
deployed with Heroku prebuilt buildpacks, i.e.
https://github.com/thenovices/heroku-buildpack-scipy. I tried to deploy
theano using pip. It does install fine, but on import I get this:
Running `bash` attached to terminal... up, run.7976
~ $ pip install -e git://github.com/Theano/Theano.git#egg=theano
pip install -e git://github.com/Theano/Theano.git#egg=theano
Obtaining theano from git+git://github.com/Theano/Theano.git#egg=theano
Cloning git://github.com/Theano/Theano.git to ./src/theano
warning: manifest_maker: MANIFEST.in, line 8: 'recursive-include'
expects <dir> <pattern1> <pattern2> ...
Requirement already satisfied (use --upgrade to upgrade): numpy>=1.5.0 in
./.heroku/python/lib/python2.7/site-packages (from theano)
Requirement already satisfied (use --upgrade to upgrade): scipy>=0.7.2 in
./.heroku/python/lib/python2.7/site-packages (from theano)
Installing collected packages: theano
Running setup.py develop for theano
warning: manifest_maker: MANIFEST.in, line 8: 'recursive-include'
expects <dir> <pattern1> <pattern2> ...
Creating /app/.heroku/python/lib/python2.7/site-packages/Theano.egg-link
(link to .)
Adding Theano 0.6.0 to easy-install.pth file
Installing theano-cache script to /app/.heroku/python/bin
Installing theano-nose script to /app/.heroku/python/bin
Installing theano-test script to /app/.heroku/python/bin
Installed /app/src/theano
Successfully installed theano-0.6.0
~ $ python
python
Python 2.7.8 (default, Jul 9 2014, 20:47:08)
[GCC 4.4.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
import theano
00001 #include <Python.h>
00002 #include "structmember.h"
00003 #include <sys/time.h>
00004
00005 // Old Python compatibility from here:
00006 // http://www.python.org/dev/peps/pep-0353/
00007 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
00008 typedef int Py_ssize_t;
00009 #define PY_SSIZE_T_MAX INT_MAX
00010 #define PY_SSIZE_T_MIN INT_MIN
00011 // This one was taken from:
00012 //
http://svn.python.org/projects/python/trunk/Modules/_ctypes/ctypes.h
00013 #define PyNumber_AsSsize_t(ob, exc) PyInt_AsLong(ob)
00014 #endif
00015
00016 #if PY_VERSION_HEX >= 0x03000000
00017 #include "numpy/npy_3kcompat.h"
00018 #define PyCObject_AsVoidPtr NpyCapsule_AsVoidPtr
00019 #define PyCObject_GetDesc NpyCapsule_GetDesc
00020 #define PyCObject_Check NpyCapsule_Check
00021 #endif
00022
00023 #ifndef Py_TYPE
00024 #define Py_TYPE(obj) obj->ob_type
00025 #endif
00026
00027 /**
00028
00029 TODO:
00030 - Check max supported depth of recursion
00031 - CLazyLinker should add context information to errors caught
during evaluation. Say what node we were on, add the traceback attached to
the node.
00032 - Clear containers of fully-useed intermediate results if allow_gc
is 1
00033 - Add timers for profiling
00034 - Add support for profiling space used.
00035
00036
00037 */
00038 static double pytime(const struct timeval * tv)
00039 {
00040 struct timeval t;
00041 if (!tv)
00042 {
00043 tv = &t;
00044 gettimeofday(&t, NULL);
00045 }
00046 return (double) tv->tv_sec + (double) tv->tv_usec / 1000000.0;
00047 }
00048
00049 /**
00050 Helper routine to convert a PyList of integers to a c array of
integers.
00051 */
00052 static int unpack_list_of_ssize_t(PyObject * pylist, Py_ssize_t **
dst, Py_ssize_t *len,
00053 const char* kwname)
00054 {
00055 Py_ssize_t buflen, *buf;
00056 if (!PyList_Check(pylist))
00057 {
00058 PyErr_Format(PyExc_TypeError, "%s must be list", kwname);
00059 return -1;
00060 }
00061 assert (NULL == *dst);
00062 *len = buflen = PyList_Size(pylist);
00063 *dst = buf = (Py_ssize_t*)calloc(buflen, sizeof(Py_ssize_t));
00064 assert(buf);
00065 for (int ii = 0; ii < buflen; ++ii)
00066 {
00067 PyObject * el_i = PyList_GetItem(pylist, ii);
00068 Py_ssize_t n_i = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00069 if (PyErr_Occurred())
00070 {
00071 free(buf);
00072 *dst = NULL;
00073 return -1;
00074 }
00075 buf[ii] = n_i;
00076 }
00077 return 0;
00078 }
00079
00080 /**
00081
00082 CLazyLinker
00083
00084
00085 */
00086 typedef struct {
00087 PyObject_HEAD
00088 /* Type-specific fields go here. */
00089 PyObject * nodes; // the python list of nodes
00090 PyObject * thunks; // python list of thunks
00091 PyObject * pre_call_clear; //list of cells to clear on call.
00092 int allow_gc;
00093 Py_ssize_t n_applies;
00094 int n_vars; // number of variables in the graph
00095 int * var_computed; // 1 or 0 for every variable
00096 PyObject ** var_computed_cells;
00097 PyObject ** var_value_cells;
00098 Py_ssize_t **dependencies; // list of vars dependencies for GC
00099 Py_ssize_t *n_dependencies;
00100
00101 Py_ssize_t n_output_vars;
00102 Py_ssize_t * output_vars; // variables that *must* be evaluated
by call
00103
00104 int * is_lazy; // 1 or 0 for every thunk
00105
00106 Py_ssize_t * var_owner; // nodes[[var_owner[var_idx]]] is
var[var_idx]->owner
00107 int * var_has_owner; // 1 or 0
00108
00109 Py_ssize_t * node_n_inputs;
00110 Py_ssize_t * node_n_outputs;
00111 Py_ssize_t ** node_inputs;
00112 Py_ssize_t ** node_outputs;
00113 Py_ssize_t * node_inputs_outputs_base; // node_inputs and
node_outputs point into this
00114 Py_ssize_t * node_n_prereqs;
00115 Py_ssize_t ** node_prereqs;
00116
00117 Py_ssize_t * update_storage; // input cells to update with the
last outputs in output_vars
00118 Py_ssize_t n_updates;
00119
00120 void ** thunk_cptr_fn;
00121 void ** thunk_cptr_data;
00122 PyObject * call_times;
00123 PyObject * call_counts;
00124 int do_timing;
00125 int need_update_inputs;
00126 int position_of_error; // -1 for no error, otw the index into
`thunks` that failed.
00127 } CLazyLinker;
00128
00129
00130 static void
00131 CLazyLinker_dealloc(PyObject* _self)
00132 {
00133 CLazyLinker* self = (CLazyLinker *) _self;
00134 free(self->thunk_cptr_fn);
00135 free(self->thunk_cptr_data);
00136
00137 free(self->is_lazy);
00138
00139 free(self->update_storage);
00140
00141 if (self->node_n_prereqs)
00142 {
00143 for (int i = 0; i < self->n_applies; ++i)
00144 {
00145 free(self->node_prereqs[i]);
00146 }
00147 }
00148 free(self->node_n_prereqs);
00149 free(self->node_prereqs);
00150 free(self->node_inputs_outputs_base);
00151 free(self->node_n_inputs);
00152 free(self->node_n_outputs);
00153 free(self->node_inputs);
00154 free(self->node_outputs);
00155
00156 if (self->dependencies)
00157 {
00158 for (int i = 0; i < self->n_vars; ++i)
00159 {
00160 free(self->dependencies[i]);
00161 }
00162 free(self->dependencies);
00163 free(self->n_dependencies);
00164 }
00165
00166 free(self->var_owner);
00167 free(self->var_has_owner);
00168 free(self->var_computed);
00169 if (self->var_computed_cells)
00170 {
00171 for (int i = 0; i < self->n_vars; ++i)
00172 {
00173 Py_DECREF(self->var_computed_cells[i]);
00174 Py_DECREF(self->var_value_cells[i]);
00175 }
00176 }
00177 free(self->var_computed_cells);
00178 free(self->var_value_cells);
00179 free(self->output_vars);
00180
00181 Py_XDECREF(self->nodes);
00182 Py_XDECREF(self->thunks);
00183 Py_XDECREF(self->call_times);
00184 Py_XDECREF(self->call_counts);
00185 Py_XDECREF(self->pre_call_clear);
00186 Py_TYPE(self)->tp_free((PyObject*)self);
00187 }
00188 static PyObject *
00189 CLazyLinker_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
00190 {
00191 CLazyLinker *self;
00192
00193 self = (CLazyLinker *)type->tp_alloc(type, 0);
00194 if (self != NULL) {
00195 self->nodes = NULL;
00196 self->thunks = NULL;
00197 self->pre_call_clear = NULL;
00198
00199 self->allow_gc = 1;
00200 self->n_applies = 0;
00201 self->n_vars = 0;
00202 self->var_computed = NULL;
00203 self->var_computed_cells = NULL;
00204 self->var_value_cells = NULL;
00205 self->dependencies = NULL;
00206 self->n_dependencies = NULL;
00207
00208 self->n_output_vars = 0;
00209 self->output_vars = NULL;
00210
00211 self->is_lazy = NULL;
00212
00213 self->var_owner = NULL;
00214 self->var_has_owner = NULL;
00215
00216 self->node_n_inputs = NULL;
00217 self->node_n_outputs = NULL;
00218 self->node_inputs = NULL;
00219 self->node_outputs = NULL;
00220 self->node_inputs_outputs_base = NULL;
00221 self->node_prereqs = NULL;
00222 self->node_n_prereqs = NULL;
00223
00224 self->update_storage = NULL;
00225 self->n_updates = 0;
00226
00227 self->thunk_cptr_data = NULL;
00228 self->thunk_cptr_fn = NULL;
00229 self->call_times = NULL;
00230 self->call_counts = NULL;
00231 self->do_timing = 0;
00232
00233 self->need_update_inputs = 0;
00234 self->position_of_error = -1;
00235 }
00236 return (PyObject *)self;
00237 }
00238
00239 static int
00240 CLazyLinker_init(CLazyLinker *self, PyObject *args, PyObject *kwds)
00241 {
00242 static char *kwlist[] = {
00243 (char*)"nodes",
00244 (char*)"thunks",
00245 (char*)"pre_call_clear",
00246 (char*)"allow_gc",
00247 (char*)"call_counts",
00248 (char*)"call_times",
00249 (char*)"compute_map_list",
00250 (char*)"storage_map_list",
00251 (char*)"base_input_output_list",
00252 (char*)"node_n_inputs",
00253 (char*)"node_n_outputs",
00254 (char*)"node_input_offset",
00255 (char*)"node_output_offset",
00256 (char*)"var_owner",
00257 (char*)"is_lazy_list",
00258 (char*)"output_vars",
00259 (char*)"node_prereqs",
00260 (char*)"node_output_size",
00261 (char*)"update_storage",
00262 (char*)"dependencies",
00263 NULL};
00264
00265 PyObject *compute_map_list=NULL,
00266 *storage_map_list=NULL,
00267 *base_input_output_list=NULL,
00268 *node_n_inputs=NULL,
00269 *node_n_outputs=NULL,
00270 *node_input_offset=NULL,
00271 *node_output_offset=NULL,
00272 *var_owner=NULL,
00273 *is_lazy=NULL,
00274 *output_vars=NULL,
00275 *node_prereqs=NULL,
00276 *node_output_size=NULL,
00277 *update_storage=NULL,
00278 *dependencies=NULL;
00279
00280 assert(!self->nodes);
00281 if (! PyArg_ParseTupleAndKeywords(args, kwds,
"OOOiOOOOOOOOOOOOOOOO", kwlist,
00282 &self->nodes,
00283 &self->thunks,
00284 &self->pre_call_clear,
00285 &self->allow_gc,
00286 &self->call_counts,
00287 &self->call_times,
00288 &compute_map_list,
00289 &storage_map_list,
00290 &base_input_output_list,
00291 &node_n_inputs,
00292 &node_n_outputs,
00293 &node_input_offset,
00294 &node_output_offset,
00295 &var_owner,
00296 &is_lazy,
00297 &output_vars,
00298 &node_prereqs,
00299 &node_output_size,
00300 &update_storage,
00301 &dependencies
00302 ))
00303 return -1;
00304 Py_INCREF(self->nodes);
00305 Py_INCREF(self->thunks);
00306 Py_INCREF(self->pre_call_clear);
00307 Py_INCREF(self->call_counts);
00308 Py_INCREF(self->call_times);
00309
00310 Py_ssize_t n_applies = PyList_Size(self->nodes);
00311
00312 self->n_applies = n_applies;
00313 self->n_vars = PyList_Size(var_owner);
00314
00315 if (PyList_Size(self->thunks) != n_applies) return -1;
00316 if (PyList_Size(self->call_counts) != n_applies) return -1;
00317 if (PyList_Size(self->call_times) != n_applies) return -1;
00318
00319 // allocated and initialize thunk_cptr_data and thunk_cptr_fn
00320 if (n_applies)
00321 {
00322 self->thunk_cptr_data = (void**)calloc(n_applies, sizeof(
void*));
00323 self->thunk_cptr_fn = (void**)calloc(n_applies, sizeof(void
*));
00324 self->is_lazy = (int*)calloc(n_applies, sizeof(int));
00325 self->node_prereqs = (Py_ssize_t**)calloc(n_applies, sizeof(
Py_ssize_t*));
00326 self->node_n_prereqs = (Py_ssize_t*)calloc(n_applies, sizeof
(Py_ssize_t));
00327 assert(self->node_prereqs);
00328 assert(self->node_n_prereqs);
00329 assert(self->is_lazy);
00330 assert(self->thunk_cptr_fn);
00331 assert(self->thunk_cptr_data);
00332
00333 for (int i = 0; i < n_applies; ++i)
00334 {
00335 PyObject * thunk = PyList_GetItem(self->thunks, i);
00336 //thunk is borrowed
00337 if (PyObject_HasAttrString(thunk, "cthunk"))
00338 {
00339 PyObject * cthunk = PyObject_GetAttrString(thunk,
"cthunk");
00340 //new reference
00341 assert (cthunk && PyCObject_Check(cthunk));
00342 self->thunk_cptr_fn[i] = PyCObject_AsVoidPtr(cthunk
);
00343 self->thunk_cptr_data[i] = PyCObject_GetDesc(cthunk
);
00344 Py_DECREF(cthunk);
00345 // cthunk is kept alive by membership in
self->thunks
00346 }
00347
00348 PyObject * el_i = PyList_GetItem(is_lazy, i);
00349 self->is_lazy[i] = PyNumber_AsSsize_t(el_i, NULL);
00350
00351 /* now get the prereqs */
00352 el_i = PyList_GetItem(node_prereqs, i);
00353 assert (PyList_Check(el_i));
00354 self->node_n_prereqs[i] = PyList_Size(el_i);
00355 if (self->node_n_prereqs[i])
00356 {
00357 self->node_prereqs[i] = (Py_ssize_t*)malloc(
00358 PyList_Size(el_i)*sizeof(Py_ssize_t));
00359 for (int j = 0; j < PyList_Size(el_i); ++j)
00360 {
00361 PyObject * el_ij = PyList_GetItem(el_i, j);
00362 Py_ssize_t N = PyNumber_AsSsize_t(el_ij,
PyExc_IndexError);
00363 if (PyErr_Occurred())
00364 return -1;
00365 // N < n. variables
00366 assert(N < PyList_Size(var_owner));
00367 self->node_prereqs[i][j] = N;
00368 }
00369 }
00370 }
00371 }
00372 if (PyList_Check(base_input_output_list))
00373 {
00374 Py_ssize_t n_inputs_outputs_base = PyList_Size(
base_input_output_list);
00375 self->node_inputs_outputs_base = (Py_ssize_t*)calloc(
n_inputs_outputs_base,sizeof(Py_ssize_t));
00376 assert(self->node_inputs_outputs_base);
00377 for (int i = 0; i < n_inputs_outputs_base; ++i)
00378 {
00379 PyObject *el_i = PyList_GetItem(base_input_output_list,
i);
00380 Py_ssize_t idx = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00381 if (PyErr_Occurred()) return -1;
00382 self->node_inputs_outputs_base[i] = idx;
00383 }
00384 self->node_n_inputs = (Py_ssize_t*)calloc(n_applies,sizeof(
Py_ssize_t));
00385 assert(self->node_n_inputs);
00386 self->node_n_outputs = (Py_ssize_t*)calloc(n_applies,sizeof(
Py_ssize_t));
00387 assert(self->node_n_outputs);
00388 self->node_inputs = (Py_ssize_t**)calloc(n_applies,sizeof(
Py_ssize_t*));
00389 assert(self->node_inputs);
00390 self->node_outputs = (Py_ssize_t**)calloc(n_applies,sizeof(
Py_ssize_t*));
00391 assert(self->node_outputs);
00392 for (int i = 0; i < n_applies; ++i)
00393 {
00394 Py_ssize_t N;
00395 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_inputs, i),
PyExc_IndexError);
00396 if (PyErr_Occurred()) return -1;
00397 assert (N <= n_inputs_outputs_base);
00398 self->node_n_inputs[i] = N;
00399 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_outputs, i
),PyExc_IndexError);
00400 if (PyErr_Occurred()) return -1;
00401 assert (N <= n_inputs_outputs_base);
00402 self->node_n_outputs[i] = N;
00403 N = PyNumber_AsSsize_t(PyList_GetItem(node_input_offset,
i),PyExc_IndexError);
00404 if (PyErr_Occurred()) return -1;
00405 assert (N <= n_inputs_outputs_base);
00406 self->node_inputs[i] = &self->node_inputs_outputs_base[N
];
00407 N = PyNumber_AsSsize_t(PyList_GetItem(node_output_offset
, i),PyExc_IndexError);
00408 if (PyErr_Occurred()) return -1;
00409 assert (N <= n_inputs_outputs_base);
00410 self->node_outputs[i] = &self->node_inputs_outputs_base[
N];
00411 }
00412 }
00413 else
00414 {
00415 PyErr_SetString(PyExc_TypeError, "base_input_output_list
must be list");
00416 return -1;
00417 }
00418
00419 // allocation for var_owner
00420 if (PyList_Check(var_owner))
00421 {
00422 self->var_owner = (Py_ssize_t*)calloc(self->n_vars,sizeof(
Py_ssize_t));
00423 self->var_has_owner = (int*)calloc(self->n_vars,sizeof(int
));
00424 self->var_computed = (int*)calloc(self->n_vars,sizeof(int));
00425 self->var_computed_cells = (PyObject**)calloc(self->n_vars,
sizeof(PyObject*));
00426 self->var_value_cells = (PyObject**)calloc(self->n_vars,
sizeof(PyObject*));
00427 for (int i = 0; i < self->n_vars; ++i)
00428 {
00429 PyObject * el_i = PyList_GetItem(var_owner, i);
00430 if (el_i == Py_None)
00431 {
00432 self->var_has_owner[i] = 0;
00433 }
00434 else
00435 {
00436 Py_ssize_t N = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00437 if (PyErr_Occurred()) return -1;
00438 assert (N <= n_applies);
00439 self->var_owner[i] = N;
00440 self->var_has_owner[i] = 1;
00441 }
00442 self->var_computed_cells[i] = PyList_GetItem(
compute_map_list, i);
00443 Py_INCREF(self->var_computed_cells[i]);
00444 self->var_value_cells[i] = PyList_GetItem(
storage_map_list, i);
00445 Py_INCREF(self->var_value_cells[i]);
00446 }
00447 }
00448 else
00449 {
00450 PyErr_SetString(PyExc_TypeError, "var_owner must be list");
00451 return -1;
00452 }
00453
00454 if (dependencies != Py_None)
00455 {
00456 self->dependencies = (Py_ssize_t**)calloc(self->n_vars,
sizeof(Py_ssize_t *));
00457 self->n_dependencies = (Py_ssize_t*)calloc(self->n_vars,
sizeof(Py_ssize_t));
00458 assert(self->dependencies);
00459 assert(self->n_dependencies);
00460
00461 for (int i = 0; i < self->n_vars; ++i)
00462 {
00463 PyObject *tmp = PyList_GetItem(dependencies, i);
00464 // refcounting - tmp is borrowed
00465 if (unpack_list_of_ssize_t(tmp, &self->dependencies[i],
&self->n_dependencies[i],
00466 "dependencies"))
00467 return -1;
00468 }
00469 }
00470
00471 if (unpack_list_of_ssize_t(output_vars, &self->output_vars, &
self->n_output_vars,
00472 "output_vars"))
00473 return -1;
00474 for (int i = 0; i < self->n_output_vars; ++i)
00475 {
00476 assert(self->output_vars[i] < self->n_vars);
00477 }
00478 if (unpack_list_of_ssize_t(update_storage, &self->update_storage
, &self->n_updates,
00479 "updates_storage"))
00480 return -1;
00481 return 0;
00482 }
00483 static void set_position_of_error(CLazyLinker * self, int owner_idx)
00484 {
00485 if (self->position_of_error == -1)
00486 {
00487 self->position_of_error = owner_idx;
00488 }
00489 }
00490 static PyObject * pycall(CLazyLinker * self, Py_ssize_t node_idx,
int verbose)
00491 {
00492 // call thunk to see which inputs it wants
00493 PyObject * thunk = PyList_GetItem(self->thunks, node_idx);
00494 // refcounting - thunk is borrowed
00495 PyObject * rval = NULL;
00496 if (self->do_timing)
00497 {
00498 double t0 = pytime(NULL);
00499 if (verbose) fprintf(stderr, "calling via Python (node %i)\n",
(int)node_idx);
00500 rval = PyObject_CallObject(thunk, NULL);
00501 if (rval)
00502 {
00503 double t1 = pytime(NULL);
00504 double ti = PyFloat_AsDouble(
00505 PyList_GetItem(self->call_times, node_idx
));
00506 PyList_SetItem(self->call_times, node_idx,
00507 PyFloat_FromDouble(t1 - t0 + ti));
00508 PyObject * count = PyList_GetItem(self->call_counts,
node_idx);
00509 long icount = PyInt_AsLong(count);
00510 PyList_SetItem(self->call_counts, node_idx,
00511 PyInt_FromLong(icount + 1));
00512 }
00513 }
00514 else
00515 {
00516 if (verbose)
00517 {
00518 fprintf(stderr, "calling via Python (node %i)\n", (int)
node_idx);
00519 }
00520 rval = PyObject_CallObject(thunk, NULL);
00521 }
00522 return rval;
00523 }
00524 static int c_call(CLazyLinker * self, Py_ssize_t node_idx, int
verbose)
00525 {
00526 void * ptr_addr = self->thunk_cptr_fn[node_idx];
00527 int (*fn)(void*) = (int (*)(void*))(ptr_addr);
00528 if (verbose) fprintf(stderr, "calling non-lazy shortcut (node
%i)\n", (int)node_idx);
00529 int err = 0;
00530 if (self->do_timing)
00531 {
00532 double t0 = pytime(NULL);
00533 err = fn(self->thunk_cptr_data[node_idx]);
00534 double t1 = pytime(NULL);
00535 double ti = PyFloat_AsDouble(PyList_GetItem(self->call_times,
node_idx));
00536 PyList_SetItem(self->call_times, node_idx, PyFloat_FromDouble(t1
- t0 + ti));
00537 PyObject * count = PyList_GetItem(self->call_counts, node_idx
);
00538 long icount = PyInt_AsLong(count);
00539 PyList_SetItem(self->call_counts, node_idx, PyInt_FromLong(
icount+1));
00540 }
00541 else
00542 {
00543 err = fn(self->thunk_cptr_data[node_idx]);
00544 }
00545
00546 if (err)
00547 {
00548 // cast the argument to a PyList (as described near line 226
of cc.py)
00549 PyObject * __ERROR = ((PyObject**)self->thunk_cptr_data[
node_idx])[0];
00550 assert (PyList_Check(__ERROR));
00551 assert (PyList_Size(__ERROR) == 3);
00552 PyObject * err_type = PyList_GetItem(__ERROR, 0); //stolen ref
00553 PyObject * err_msg = PyList_GetItem(__ERROR, 1); //stolen ref
00554 PyObject * err_trace = PyList_GetItem(__ERROR, 2); //stolen
ref
00555 PyList_SET_ITEM(__ERROR, 0, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00556 PyList_SET_ITEM(__ERROR, 1, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00557 PyList_SET_ITEM(__ERROR, 2, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00558
00559 assert(!PyErr_Occurred()); // because CLinker hid the
exception in __ERROR aka data
00560 PyErr_Restore(err_type, err_msg, err_trace); //steals refs to
args
00561 }
00562 if (err) set_position_of_error(self, node_idx);
00563 return err;
00564 }
00565 static
00566 int lazy_rec_eval(CLazyLinker * self, Py_ssize_t var_idx, PyObject*
one, PyObject*zero)
00567 {
00568 PyObject *rval = NULL;
00569 int verbose = 0;
00570 int err = 0;
00571
00572 if (verbose) fprintf(stderr, "lazy_rec computing %i\n", (int)
var_idx);
00573
00574 if (self->var_computed[var_idx] || !self->var_has_owner[var_idx])
00575 return 0;
00576
00577 Py_ssize_t owner_idx = self->var_owner[var_idx];
00578
00579 // STEP 1: compute the pre-requirements of the node
00580 // Includes input nodes for non-lazy ops.
00581 for (int i = 0; i < self->node_n_prereqs[owner_idx]; ++i)
00582 {
00583 Py_ssize_t prereq_idx = self->node_prereqs[owner_idx][i];
00584 if (!self->var_computed[prereq_idx])
00585 {
00586 err = lazy_rec_eval(self, prereq_idx, one, zero);
00587 if (err) return err;
00588 }
00589 assert (self->var_computed[prereq_idx]);
00590 }
00591
00592 // STEP 2: compute the node itself
00593 if (self->is_lazy[owner_idx])
00594 {
00595 // update the compute_map cells corresponding to the inputs
of this thunk
00596 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00597 {
00598 int in_idx = self->node_inputs[owner_idx][i];
00599 if (self->var_computed[in_idx])
00600 {
00601 Py_INCREF(one);
00602 err = PyList_SetItem(self->var_computed_cells[in_idx],
0, one);
00603 }
00604 else
00605 {
00606 Py_INCREF(zero);
00607 err = PyList_SetItem(self->var_computed_cells[in_idx],
0, zero);
00608 }
00609 if (err) goto fail;
00610 }
00611
00612 rval = pycall(self, owner_idx, verbose);
00613 // refcounting - rval is new ref
00614 //TODO: to prevent infinite loops
00615 // - consider check that a thunk does not ask for an input
that is already computed
00616 if (rval == NULL)
00617 {
00618 assert (PyErr_Occurred());
00619 err = 1;
00620 goto fail;
00621 }
00622
00623 //update the computed-ness of any output cells
00624 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00625 {
00626 int out_idx = self->node_outputs[owner_idx][i];
00627 PyObject * el_i = PyList_GetItem(self->var_computed_cells[
out_idx], 0);
00628 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00629 if (PyErr_Occurred())
00630 {
00631 err = -1;
00632 goto pyfail;
00633 }
00634 assert (N==0 || N==1);
00635 self->var_computed[out_idx] = N;
00636 }
00637 if (!self->var_computed[var_idx])
00638 {
00639 /*
00640 * If self is not computed after the call, this means
that some
00641 * inputs are needed. Compute the ones on the returned
list
00642 * and try to compute the current node again (with
recursive call).
00643 * This allows a node to request more nodes more than
once before
00644 * finally yielding a result.
00645 */
00646 if (!PyList_Check(rval))
00647 {
00648 //TODO: More helpful error to help find *which node*
made this
00649 // bad thunk
00650 PyErr_SetString(PyExc_TypeError,
00651 "lazy thunk should return a list");
00652 err = 1;
00653 goto pyfail;
00654 }
00655
00656 if (!PyList_Size(rval))
00657 {
00658 PyErr_SetString(PyExc_ValueError,
00659 "lazy thunk returned empty list
without computing output");
00660 err = 1;
00661 goto pyfail;
00662 }
00663
00664 for (int i = 0; i < PyList_Size(rval); ++i)
00665 {
00666 PyObject * el_i = PyList_GetItem(rval, i);
00667 Py_ssize_t N = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00668 if (PyErr_Occurred())
00669 {
00670 err = 1;
00671 goto pyfail;
00672 }
00673 assert (N <= self->node_n_inputs[owner_idx]);
00674 Py_ssize_t input_idx = self->node_inputs[owner_idx][N
];
00675 err = lazy_rec_eval(self, input_idx, one, zero);
00676 if (err) goto pyfail;
00677 }
00678
00679 Py_DECREF(rval);
00680 /*
00681 * We intentionally skip all the end-of-function
processing
00682 * (mark outputs, GC) as it will be performed by the call
00683 * that actually manages to compute the result.
00684 */
00685 return lazy_rec_eval(self, var_idx, one, zero);
00686 }
00687
00688 Py_DECREF(rval);
00689 }
00690 else //owner is not a lazy op. Ensure all intputs are evaluated.
00691 {
00692 // loop over inputs to owner
00693 // call lazy_rec_eval on each one that is not computed.
00694 // if there's an error, pass it up the stack
00695 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00696 {
00697 Py_ssize_t input_idx = self->node_inputs[owner_idx][i];
00698 if (!self->var_computed[input_idx])
00699 {
00700 err = lazy_rec_eval(self, input_idx, one, zero);
00701 if (err) return err;
00702 }
00703 assert (self->var_computed[input_idx]);
00704 }
00705
00706 // call the thunk for this owner.
00707 if (self->thunk_cptr_fn[owner_idx])
00708 {
00709 err = c_call(self, owner_idx, verbose);
00710 if (err) goto fail;
00711 }
00712 else
00713 {
00714 rval = pycall(self, owner_idx, verbose);
00715 //rval is new ref
00716 if (rval) //pycall returned normally (no exception)
00717 {
00718 if (rval == Py_None)
00719 {
00720 Py_DECREF(rval); //ignore a return of None
00721 }
00722 else if (PyList_Check(rval))
00723 {
00724 PyErr_SetString(PyExc_TypeError,
00725 "non-lazy thunk should return
None, not list");
00726 err = 1;
00727 goto pyfail;
00728 }
00729 else // don't know what it returned, but it wasn't
right.
00730 {
00731 PyErr_SetObject(PyExc_TypeError, rval);
00732 err = 1;
00733 // We don't release rval since we put it in the
error above
00734 goto fail;
00735 }
00736 }
00737 else // pycall returned NULL (internal error)
00738 {
00739 err = 1;
00740 goto fail;
00741 }
00742 }
00743 }
00744
00745 // loop over all outputs and mark them as computed
00746 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00747 {
00748 self->var_computed[self->node_outputs[owner_idx][i]] = 1;
00749 }
00750
00751 // Free vars that are not needed anymore
00752 if (self->allow_gc)
00753 {
00754 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00755 {
00756 int cleanup = 1;
00757 Py_ssize_t i_idx = self->node_inputs[owner_idx][i];
00758 if (!self->var_has_owner[i_idx])
00759 continue;
00760
00761 for (int j = 0; j < self->n_output_vars; ++j)
00762 {
00763 if (i_idx == self->output_vars[j])
00764 {
00765 cleanup = 0;
00766 break;
00767 }
00768 }
00769 if (!cleanup) continue;
00770
00771 for (int j = 0; j < self->n_dependencies[i_idx]; ++j)
00772 {
00773 if (!self->var_computed[self->dependencies[i_idx][j]])
00774 {
00775 cleanup = 0;
00776 break;
00777 }
00778 }
00779 if (!cleanup) continue;
00780
00781 Py_INCREF(Py_None);
00782 err = PyList_SetItem(self->var_value_cells[i_idx], 0,
Py_None);
00783 //See the Stack gc implementation for why we change it to 2 and not
0.
00784 self->var_computed[i_idx] = 2;
00785 if (err) goto fail;
00786 }
00787 }
00788
00789 return 0;
00790 pyfail:
00791 Py_DECREF(rval);
00792 fail:
00793 set_position_of_error(self, owner_idx);
00794 return err;
00795 }
00796
00797 static PyObject *
00798 CLazyLinker_call(PyObject *_self, PyObject *args, PyObject *kwds)
00799 {
00800 CLazyLinker * self = (CLazyLinker*)_self;
00801 static char *kwlist[] = {
00802 (char*)"time_thunks",
00803 (char *)"n_calls",
00804 NULL};
00805 int n_calls=1;
00806 if (! PyArg_ParseTupleAndKeywords(args, kwds, "|ii", kwlist,
00807 &self->do_timing,
00808 &n_calls))
00809 return NULL;
00810 int err = 0;
00811 self->position_of_error = -1;
00812 // create constants used to fill the var_compute_cells
00813 PyObject * one = PyInt_FromLong(1);
00814 PyObject * zero = PyInt_FromLong(0);
00815
00816 // pre-allocate our return value
00817 Py_INCREF(Py_None);
00818 PyObject * rval = Py_None;
00819 //clear storage of pre_call_clear elements
00820 for (int call_i = 0; call_i < n_calls && (!err); ++call_i)
00821 {
00822 Py_ssize_t n_pre_call_clear = PyList_Size(self->pre_call_clear
);
00823 assert(PyList_Check(self->pre_call_clear));
00824 for (int i = 0; i < n_pre_call_clear; ++i)
00825 {
00826 PyObject * el_i = PyList_GetItem(self->pre_call_clear, i);
00827 Py_INCREF(Py_None);
00828 PyList_SetItem(el_i, 0, Py_None);
00829 }
00830 //clear the computed flag out of all non-input vars
00831 for (int i = 0; i < self->n_vars; ++i)
00832 {
00833 self->var_computed[i] = !self->var_has_owner[i];
00834 if (self->var_computed[i])
00835 {
00836 Py_INCREF(one);
00837 PyList_SetItem(self->var_computed_cells[i], 0, one);
00838 }
00839 else
00840 {
00841 Py_INCREF(zero);
00842 PyList_SetItem(self->var_computed_cells[i], 0, zero);
00843 }
00844 }
00845
00846 for (int i = 0; i < self->n_output_vars && (!err); ++i)
00847 {
00848 err = lazy_rec_eval(self, self->output_vars[i], one, zero
);
00849 }
00850
00851 if (!err)
00852 {
00853 // save references to outputs prior to updating storage
containers
00854 assert (self->n_output_vars >= self->n_updates);
00855 Py_DECREF(rval);
00856 rval = PyList_New(self->n_output_vars);
00857 for (int i = 0; i < (self->n_output_vars); ++i)
00858 {
00859 Py_ssize_t src = self->output_vars[i];
00860 PyObject * item = PyList_GetItem(self->var_value_cells
[src], 0);
00861 if (self->var_computed[src] != 1)
00862 {
00863 err = 1;
00864 PyErr_Format(PyExc_AssertionError,
00865 "The compute map of output %d should
contain "
00866 "1 at the end of execution, not %d.",
00867 i, self->var_computed[src]);
00868 break;
00869 }
00870 Py_INCREF(item);
00871 PyList_SetItem(rval, i, item);
00872 }
00873 }
00874
00875 if (!err)
00876 {
00877 // Update the inputs that have an update rule
00878 for (int i = 0; i < self->n_updates; ++i)
00879 {
00880 PyObject* tmp = PyList_GetItem(rval, self->n_output_vars
- self->n_updates + i);
00881 Py_INCREF(tmp);
00882 Py_ssize_t dst = self->update_storage[i];
00883 PyList_SetItem(self->var_value_cells[dst], 0, tmp);
00884 }
00885 }
00886 }
00887
00888 /*
00889 Clear everything that is left and not an output. This is needed
00890 for lazy evaluation since the current GC algo is too
conservative
00891 with lazy graphs.
00892 */
00893 if (self->allow_gc && !err)
00894 {
00895 for (Py_ssize_t i = 0; i < self->n_vars; ++i)
00896 {
00897 int do_cleanup = 1;
00898 if (!self->var_has_owner[i] || !self->var_computed[i])
00899 continue;
00900 for (int j = 0; j < self->n_output_vars; ++j)
00901 {
00902 if (i == self->output_vars[j])
00903 {
00904 do_cleanup = 0;
00905 break;
00906 }
00907 }
00908 if (!do_cleanup)
00909 continue;
00910 Py_INCREF(Py_None);
00911 PyList_SetItem(self->var_value_cells[i], 0, Py_None);
00912 }
00913 }
00914 Py_DECREF(one);
00915 Py_DECREF(zero);
00916 if (err)
00917 {
00918 Py_DECREF(rval);
00919 return NULL;
00920 }
00921 return rval;
00922 }
00923
00924 #if 0
00925 static PyMethodDef CLazyLinker_methods[] = {
00926 {
00927 //"name", (PyCFunction)CLazyLinker_accept, METH_VARARGS,
"Return the name, combining the first and last name"
00928 },
00929 {NULL} /* Sentinel */
00930 };
00931 #endif
00932
00933
00934 static PyObject *
00935 CLazyLinker_get_allow_gc(CLazyLinker *self, void *closure)
00936 {
00937 return PyBool_FromLong(self->allow_gc);
00938 }
00939
00940 static int
00941 CLazyLinker_set_allow_gc(CLazyLinker *self, PyObject *value, void *
closure)
00942 {
00943 if(!PyBool_Check(value))
00944 return -1;
00945
00946 if (value == Py_True)
00947 self->allow_gc = true;
00948 else
00949 self->allow_gc = false;
00950 return 0;
00951 }
00952
00953 static PyGetSetDef CLazyLinker_getset[] = {
00954 {(char*)"allow_gc",
00955 (getter)CLazyLinker_get_allow_gc,
00956 (setter)CLazyLinker_set_allow_gc,
00957 (char*)"do this function support allow_gc",
00958 NULL},
00959 {NULL, NULL, NULL, NULL} /* Sentinel */
00960 };
00961 static PyMemberDef CLazyLinker_members[] = {
00962 {(char*)"nodes", T_OBJECT_EX, offsetof(CLazyLinker, nodes), 0,
00963 (char*)"list of nodes"},
00964 {(char*)"thunks", T_OBJECT_EX, offsetof(CLazyLinker, thunks), 0,
00965 (char*)"list of thunks in program"},
00966 {(char*)"call_counts", T_OBJECT_EX, offsetof(CLazyLinker,
call_counts), 0,
00967 (char*)"number of calls of each thunk"},
00968 {(char*)"call_times", T_OBJECT_EX, offsetof(CLazyLinker,
call_times), 0,
00969 (char*)"total runtime in each thunk"},
00970 {(char*)"position_of_error", T_INT, offsetof(CLazyLinker,
position_of_error), 0,
00971 (char*)"position of failed thunk"},
00972 {(char*)"time_thunks", T_INT, offsetof(CLazyLinker, do_timing),
0,
00973 (char*)"bool: nonzero means call will time thunks"},
00974 {(char*)"need_update_inputs", T_INT, offsetof(CLazyLinker,
need_update_inputs), 0,
00975 (char*)"bool: nonzero means Function.__call__ must implement
update mechanism"},
00976 {NULL} /* Sentinel */
00977 };
00978
00979 static PyTypeObject lazylinker_ext_CLazyLinkerType = {
00980 #if defined(NPY_PY3K)
00981 PyVarObject_HEAD_INIT(NULL, 0)
00982 #else
00983 PyObject_HEAD_INIT(NULL)
00984 0, /*ob_size*/
00985 #endif
00986 "lazylinker_ext.CLazyLinker", /*tp_name*/
00987 sizeof(CLazyLinker), /*tp_basicsize*/
00988 0, /*tp_itemsize*/
00989 CLazyLinker_dealloc, /*tp_dealloc*/
00990 0, /*tp_print*/
00991 0, /*tp_getattr*/
00992 0, /*tp_setattr*/
00993 0, /*tp_compare*/
00994 0, /*tp_repr*/
00995 0, /*tp_as_number*/
00996 0, /*tp_as_sequence*/
00997 0, /*tp_as_mapping*/
00998 0, /*tp_hash */
00999 CLazyLinker_call, /*tp_call*/
01000 0, /*tp_str*/
01001 0, /*tp_getattro*/
01002 0, /*tp_setattro*/
01003 0, /*tp_as_buffer*/
01004 Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
01005 "CLazyLinker object", /* tp_doc */
01006 0, /* tp_traverse */
01007 0, /* tp_clear */
01008 0, /* tp_richcompare */
01009 0, /* tp_weaklistoffset */
01010 0, /* tp_iter */
01011 0, /* tp_iternext */
01012 0,//CLazyLinker_methods, /* tp_methods */
01013 CLazyLinker_members, /* tp_members */
01014 CLazyLinker_getset, /* tp_getset */
01015 0, /* tp_base */
01016 0, /* tp_dict */
01017 0, /* tp_descr_get */
01018 0, /* tp_descr_set */
01019 0, /* tp_dictoffset */
01020 (initproc)CLazyLinker_init,/* tp_init */
01021 0, /* tp_alloc */
01022 CLazyLinker_new, /* tp_new */
01023 };
01024
01025 static PyObject * get_version(PyObject *dummy, PyObject *args)
01026 {
01027 PyObject *result = PyFloat_FromDouble(0.21);
01028 return result;
01029 }
01030
01031 static PyMethodDef lazylinker_ext_methods[] = {
01032 {"get_version", get_version, METH_VARARGS, "Get extension
version."},
01033 {NULL, NULL, 0, NULL} /* Sentinel */
01034 };
01035
01036 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
01037 #define PyMODINIT_FUNC void
01038 #endif
01039
01040 #if defined(NPY_PY3K)
01041 static struct PyModuleDef moduledef = {
01042 PyModuleDef_HEAD_INIT,
01043 "lazylinker_ext",
01044 NULL,
01045 -1,
01046 lazylinker_ext_methods,
01047 NULL,
01048 NULL,
01049 NULL,
01050 NULL
01051 };
01052 #endif
01053 #if defined(NPY_PY3K)
01054 #define RETVAL m
01055 PyMODINIT_FUNC
01056 PyInit_lazylinker_ext(void) {
01057 #else
01058 #define RETVAL
01059 PyMODINIT_FUNC
01060 initlazylinker_ext(void)
01061 {
01062 #endif
01063 PyObject* m;
01064
01065 lazylinker_ext_CLazyLinkerType.tp_new = PyType_GenericNew;
01066 if (PyType_Ready(&lazylinker_ext_CLazyLinkerType) < 0)
01067 return RETVAL;
01068 #if defined(NPY_PY3K)
01069 m = PyModule_Create(&moduledef);
01070 #else
01071 m = Py_InitModule3("lazylinker_ext", lazylinker_ext_methods,
01072 "Example module that creates an extension
type.");
01073 #endif
01074 Py_INCREF(&lazylinker_ext_CLazyLinkerType);
01075 PyModule_AddObject(m, "CLazyLinker", (PyObject *)&
lazylinker_ext_CLazyLinkerType);
01076
01077 return RETVAL;
01078 }
01079
01080
===============================
Problem occurred during compilation with the command line below:
/usr/bin/g++ -shared -g -march=core2 -mcx16 -msahf -mpopcnt -msse4.2 --param
l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=256 -
mtune=core2 -D NPY_NO_DEPRECATED_API=NPY_1_7
_API_VERSION -m64 -fPIC -I/app/.heroku/python/lib/python2.7/site-packages/
numpy/core/include -I/app/.heroku/python/include/python2.7 -o /app/.theano/
compiledir_Linux-3.13--generic-x86_64-with-deb
ian-squeeze-sid--2.7.8-64/lazylinker_ext/lazylinker_ext.so /app/.theano/
compiledir_Linux-3.13--generic-x86_64-with-debian-squeeze-sid--2.7.8-64/
lazylinker_ext/mod.cpp -L/app/.heroku/python/lib -l
python2.7
/usr/bin/ld: /app/.heroku/python/lib/libpython2.7.a(abstract.o): relocation
R_X86_64_32 against `.rodata.str1.8' can not be used when making a shared
object; recompile with -fPIC
/app/.heroku/python/lib/libpython2.7.a: could not read symbols: Bad value
collect2: ld returned 1 exit status
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/app/src/theano/theano/__init__.py", line 55, in <module>
from theano.compile import \
File "/app/src/theano/theano/compile/__init__.py", line 9, in <module>
from theano.compile.function_module import *
File "/app/src/theano/theano/compile/function_module.py", line 18, in
<module>
import theano.compile.mode
File "/app/src/theano/theano/compile/mode.py", line 11, in <module>
import theano.gof.vm
File "/app/src/theano/theano/gof/vm.py", line 568, in <module>
import lazylinker_c
File "/app/src/theano/theano/gof/lazylinker_c.py", line 116, in <module>
preargs=args)
File "/app/src/theano/theano/gof/cmodule.py", line 1952, in compile_str
(status, compile_stderr.replace('\n', '. ')))
Exception: Compilation failed (return status=1): /usr/bin/ld:
/app/.heroku/python/lib/libpython2.7.a(abstract.o): relocation R_X86_64_32
against `.rodata.str1.8' can not be used when making a sha
red object; recompile with -fPIC. /app/.heroku/python/lib/libpython2.7.a:
could not read symbols: Bad value. collect2: ld returned 1 exit status.
Has anyone tried to deploy theano on Heroku? Numpy and Scipy are usually
deployed with Heroku prebuilt buildpacks, i.e.
https://github.com/thenovices/heroku-buildpack-scipy. I tried to deploy
theano using pip. It does install fine, but on import I get this:
Running `bash` attached to terminal... up, run.7976
~ $ pip install -e git://github.com/Theano/Theano.git#egg=theano
pip install -e git://github.com/Theano/Theano.git#egg=theano
Obtaining theano from git+git://github.com/Theano/Theano.git#egg=theano
Cloning git://github.com/Theano/Theano.git to ./src/theano
warning: manifest_maker: MANIFEST.in, line 8: 'recursive-include'
expects <dir> <pattern1> <pattern2> ...
Requirement already satisfied (use --upgrade to upgrade): numpy>=1.5.0 in
./.heroku/python/lib/python2.7/site-packages (from theano)
Requirement already satisfied (use --upgrade to upgrade): scipy>=0.7.2 in
./.heroku/python/lib/python2.7/site-packages (from theano)
Installing collected packages: theano
Running setup.py develop for theano
warning: manifest_maker: MANIFEST.in, line 8: 'recursive-include'
expects <dir> <pattern1> <pattern2> ...
Creating /app/.heroku/python/lib/python2.7/site-packages/Theano.egg-link
(link to .)
Adding Theano 0.6.0 to easy-install.pth file
Installing theano-cache script to /app/.heroku/python/bin
Installing theano-nose script to /app/.heroku/python/bin
Installing theano-test script to /app/.heroku/python/bin
Installed /app/src/theano
Successfully installed theano-0.6.0
~ $ python
python
Python 2.7.8 (default, Jul 9 2014, 20:47:08)
[GCC 4.4.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
import theano
import theano
===============================00001 #include <Python.h>
00002 #include "structmember.h"
00003 #include <sys/time.h>
00004
00005 // Old Python compatibility from here:
00006 // http://www.python.org/dev/peps/pep-0353/
00007 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
00008 typedef int Py_ssize_t;
00009 #define PY_SSIZE_T_MAX INT_MAX
00010 #define PY_SSIZE_T_MIN INT_MIN
00011 // This one was taken from:
00012 //
http://svn.python.org/projects/python/trunk/Modules/_ctypes/ctypes.h
00013 #define PyNumber_AsSsize_t(ob, exc) PyInt_AsLong(ob)
00014 #endif
00015
00016 #if PY_VERSION_HEX >= 0x03000000
00017 #include "numpy/npy_3kcompat.h"
00018 #define PyCObject_AsVoidPtr NpyCapsule_AsVoidPtr
00019 #define PyCObject_GetDesc NpyCapsule_GetDesc
00020 #define PyCObject_Check NpyCapsule_Check
00021 #endif
00022
00023 #ifndef Py_TYPE
00024 #define Py_TYPE(obj) obj->ob_type
00025 #endif
00026
00027 /**
00028
00029 TODO:
00030 - Check max supported depth of recursion
00031 - CLazyLinker should add context information to errors caught
during evaluation. Say what node we were on, add the traceback attached to
the node.
00032 - Clear containers of fully-useed intermediate results if allow_gc
is 1
00033 - Add timers for profiling
00034 - Add support for profiling space used.
00035
00036
00037 */
00038 static double pytime(const struct timeval * tv)
00039 {
00040 struct timeval t;
00041 if (!tv)
00042 {
00043 tv = &t;
00044 gettimeofday(&t, NULL);
00045 }
00046 return (double) tv->tv_sec + (double) tv->tv_usec / 1000000.0;
00047 }
00048
00049 /**
00050 Helper routine to convert a PyList of integers to a c array of
integers.
00051 */
00052 static int unpack_list_of_ssize_t(PyObject * pylist, Py_ssize_t **
dst, Py_ssize_t *len,
00053 const char* kwname)
00054 {
00055 Py_ssize_t buflen, *buf;
00056 if (!PyList_Check(pylist))
00057 {
00058 PyErr_Format(PyExc_TypeError, "%s must be list", kwname);
00059 return -1;
00060 }
00061 assert (NULL == *dst);
00062 *len = buflen = PyList_Size(pylist);
00063 *dst = buf = (Py_ssize_t*)calloc(buflen, sizeof(Py_ssize_t));
00064 assert(buf);
00065 for (int ii = 0; ii < buflen; ++ii)
00066 {
00067 PyObject * el_i = PyList_GetItem(pylist, ii);
00068 Py_ssize_t n_i = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00069 if (PyErr_Occurred())
00070 {
00071 free(buf);
00072 *dst = NULL;
00073 return -1;
00074 }
00075 buf[ii] = n_i;
00076 }
00077 return 0;
00078 }
00079
00080 /**
00081
00082 CLazyLinker
00083
00084
00085 */
00086 typedef struct {
00087 PyObject_HEAD
00088 /* Type-specific fields go here. */
00089 PyObject * nodes; // the python list of nodes
00090 PyObject * thunks; // python list of thunks
00091 PyObject * pre_call_clear; //list of cells to clear on call.
00092 int allow_gc;
00093 Py_ssize_t n_applies;
00094 int n_vars; // number of variables in the graph
00095 int * var_computed; // 1 or 0 for every variable
00096 PyObject ** var_computed_cells;
00097 PyObject ** var_value_cells;
00098 Py_ssize_t **dependencies; // list of vars dependencies for GC
00099 Py_ssize_t *n_dependencies;
00100
00101 Py_ssize_t n_output_vars;
00102 Py_ssize_t * output_vars; // variables that *must* be evaluated
by call
00103
00104 int * is_lazy; // 1 or 0 for every thunk
00105
00106 Py_ssize_t * var_owner; // nodes[[var_owner[var_idx]]] is
var[var_idx]->owner
00107 int * var_has_owner; // 1 or 0
00108
00109 Py_ssize_t * node_n_inputs;
00110 Py_ssize_t * node_n_outputs;
00111 Py_ssize_t ** node_inputs;
00112 Py_ssize_t ** node_outputs;
00113 Py_ssize_t * node_inputs_outputs_base; // node_inputs and
node_outputs point into this
00114 Py_ssize_t * node_n_prereqs;
00115 Py_ssize_t ** node_prereqs;
00116
00117 Py_ssize_t * update_storage; // input cells to update with the
last outputs in output_vars
00118 Py_ssize_t n_updates;
00119
00120 void ** thunk_cptr_fn;
00121 void ** thunk_cptr_data;
00122 PyObject * call_times;
00123 PyObject * call_counts;
00124 int do_timing;
00125 int need_update_inputs;
00126 int position_of_error; // -1 for no error, otw the index into
`thunks` that failed.
00127 } CLazyLinker;
00128
00129
00130 static void
00131 CLazyLinker_dealloc(PyObject* _self)
00132 {
00133 CLazyLinker* self = (CLazyLinker *) _self;
00134 free(self->thunk_cptr_fn);
00135 free(self->thunk_cptr_data);
00136
00137 free(self->is_lazy);
00138
00139 free(self->update_storage);
00140
00141 if (self->node_n_prereqs)
00142 {
00143 for (int i = 0; i < self->n_applies; ++i)
00144 {
00145 free(self->node_prereqs[i]);
00146 }
00147 }
00148 free(self->node_n_prereqs);
00149 free(self->node_prereqs);
00150 free(self->node_inputs_outputs_base);
00151 free(self->node_n_inputs);
00152 free(self->node_n_outputs);
00153 free(self->node_inputs);
00154 free(self->node_outputs);
00155
00156 if (self->dependencies)
00157 {
00158 for (int i = 0; i < self->n_vars; ++i)
00159 {
00160 free(self->dependencies[i]);
00161 }
00162 free(self->dependencies);
00163 free(self->n_dependencies);
00164 }
00165
00166 free(self->var_owner);
00167 free(self->var_has_owner);
00168 free(self->var_computed);
00169 if (self->var_computed_cells)
00170 {
00171 for (int i = 0; i < self->n_vars; ++i)
00172 {
00173 Py_DECREF(self->var_computed_cells[i]);
00174 Py_DECREF(self->var_value_cells[i]);
00175 }
00176 }
00177 free(self->var_computed_cells);
00178 free(self->var_value_cells);
00179 free(self->output_vars);
00180
00181 Py_XDECREF(self->nodes);
00182 Py_XDECREF(self->thunks);
00183 Py_XDECREF(self->call_times);
00184 Py_XDECREF(self->call_counts);
00185 Py_XDECREF(self->pre_call_clear);
00186 Py_TYPE(self)->tp_free((PyObject*)self);
00187 }
00188 static PyObject *
00189 CLazyLinker_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
00190 {
00191 CLazyLinker *self;
00192
00193 self = (CLazyLinker *)type->tp_alloc(type, 0);
00194 if (self != NULL) {
00195 self->nodes = NULL;
00196 self->thunks = NULL;
00197 self->pre_call_clear = NULL;
00198
00199 self->allow_gc = 1;
00200 self->n_applies = 0;
00201 self->n_vars = 0;
00202 self->var_computed = NULL;
00203 self->var_computed_cells = NULL;
00204 self->var_value_cells = NULL;
00205 self->dependencies = NULL;
00206 self->n_dependencies = NULL;
00207
00208 self->n_output_vars = 0;
00209 self->output_vars = NULL;
00210
00211 self->is_lazy = NULL;
00212
00213 self->var_owner = NULL;
00214 self->var_has_owner = NULL;
00215
00216 self->node_n_inputs = NULL;
00217 self->node_n_outputs = NULL;
00218 self->node_inputs = NULL;
00219 self->node_outputs = NULL;
00220 self->node_inputs_outputs_base = NULL;
00221 self->node_prereqs = NULL;
00222 self->node_n_prereqs = NULL;
00223
00224 self->update_storage = NULL;
00225 self->n_updates = 0;
00226
00227 self->thunk_cptr_data = NULL;
00228 self->thunk_cptr_fn = NULL;
00229 self->call_times = NULL;
00230 self->call_counts = NULL;
00231 self->do_timing = 0;
00232
00233 self->need_update_inputs = 0;
00234 self->position_of_error = -1;
00235 }
00236 return (PyObject *)self;
00237 }
00238
00239 static int
00240 CLazyLinker_init(CLazyLinker *self, PyObject *args, PyObject *kwds)
00241 {
00242 static char *kwlist[] = {
00243 (char*)"nodes",
00244 (char*)"thunks",
00245 (char*)"pre_call_clear",
00246 (char*)"allow_gc",
00247 (char*)"call_counts",
00248 (char*)"call_times",
00249 (char*)"compute_map_list",
00250 (char*)"storage_map_list",
00251 (char*)"base_input_output_list",
00252 (char*)"node_n_inputs",
00253 (char*)"node_n_outputs",
00254 (char*)"node_input_offset",
00255 (char*)"node_output_offset",
00256 (char*)"var_owner",
00257 (char*)"is_lazy_list",
00258 (char*)"output_vars",
00259 (char*)"node_prereqs",
00260 (char*)"node_output_size",
00261 (char*)"update_storage",
00262 (char*)"dependencies",
00263 NULL};
00264
00265 PyObject *compute_map_list=NULL,
00266 *storage_map_list=NULL,
00267 *base_input_output_list=NULL,
00268 *node_n_inputs=NULL,
00269 *node_n_outputs=NULL,
00270 *node_input_offset=NULL,
00271 *node_output_offset=NULL,
00272 *var_owner=NULL,
00273 *is_lazy=NULL,
00274 *output_vars=NULL,
00275 *node_prereqs=NULL,
00276 *node_output_size=NULL,
00277 *update_storage=NULL,
00278 *dependencies=NULL;
00279
00280 assert(!self->nodes);
00281 if (! PyArg_ParseTupleAndKeywords(args, kwds,
"OOOiOOOOOOOOOOOOOOOO", kwlist,
00282 &self->nodes,
00283 &self->thunks,
00284 &self->pre_call_clear,
00285 &self->allow_gc,
00286 &self->call_counts,
00287 &self->call_times,
00288 &compute_map_list,
00289 &storage_map_list,
00290 &base_input_output_list,
00291 &node_n_inputs,
00292 &node_n_outputs,
00293 &node_input_offset,
00294 &node_output_offset,
00295 &var_owner,
00296 &is_lazy,
00297 &output_vars,
00298 &node_prereqs,
00299 &node_output_size,
00300 &update_storage,
00301 &dependencies
00302 ))
00303 return -1;
00304 Py_INCREF(self->nodes);
00305 Py_INCREF(self->thunks);
00306 Py_INCREF(self->pre_call_clear);
00307 Py_INCREF(self->call_counts);
00308 Py_INCREF(self->call_times);
00309
00310 Py_ssize_t n_applies = PyList_Size(self->nodes);
00311
00312 self->n_applies = n_applies;
00313 self->n_vars = PyList_Size(var_owner);
00314
00315 if (PyList_Size(self->thunks) != n_applies) return -1;
00316 if (PyList_Size(self->call_counts) != n_applies) return -1;
00317 if (PyList_Size(self->call_times) != n_applies) return -1;
00318
00319 // allocated and initialize thunk_cptr_data and thunk_cptr_fn
00320 if (n_applies)
00321 {
00322 self->thunk_cptr_data = (void**)calloc(n_applies, sizeof(
void*));
00323 self->thunk_cptr_fn = (void**)calloc(n_applies, sizeof(void
*));
00324 self->is_lazy = (int*)calloc(n_applies, sizeof(int));
00325 self->node_prereqs = (Py_ssize_t**)calloc(n_applies, sizeof(
Py_ssize_t*));
00326 self->node_n_prereqs = (Py_ssize_t*)calloc(n_applies, sizeof
(Py_ssize_t));
00327 assert(self->node_prereqs);
00328 assert(self->node_n_prereqs);
00329 assert(self->is_lazy);
00330 assert(self->thunk_cptr_fn);
00331 assert(self->thunk_cptr_data);
00332
00333 for (int i = 0; i < n_applies; ++i)
00334 {
00335 PyObject * thunk = PyList_GetItem(self->thunks, i);
00336 //thunk is borrowed
00337 if (PyObject_HasAttrString(thunk, "cthunk"))
00338 {
00339 PyObject * cthunk = PyObject_GetAttrString(thunk,
"cthunk");
00340 //new reference
00341 assert (cthunk && PyCObject_Check(cthunk));
00342 self->thunk_cptr_fn[i] = PyCObject_AsVoidPtr(cthunk
);
00343 self->thunk_cptr_data[i] = PyCObject_GetDesc(cthunk
);
00344 Py_DECREF(cthunk);
00345 // cthunk is kept alive by membership in
self->thunks
00346 }
00347
00348 PyObject * el_i = PyList_GetItem(is_lazy, i);
00349 self->is_lazy[i] = PyNumber_AsSsize_t(el_i, NULL);
00350
00351 /* now get the prereqs */
00352 el_i = PyList_GetItem(node_prereqs, i);
00353 assert (PyList_Check(el_i));
00354 self->node_n_prereqs[i] = PyList_Size(el_i);
00355 if (self->node_n_prereqs[i])
00356 {
00357 self->node_prereqs[i] = (Py_ssize_t*)malloc(
00358 PyList_Size(el_i)*sizeof(Py_ssize_t));
00359 for (int j = 0; j < PyList_Size(el_i); ++j)
00360 {
00361 PyObject * el_ij = PyList_GetItem(el_i, j);
00362 Py_ssize_t N = PyNumber_AsSsize_t(el_ij,
PyExc_IndexError);
00363 if (PyErr_Occurred())
00364 return -1;
00365 // N < n. variables
00366 assert(N < PyList_Size(var_owner));
00367 self->node_prereqs[i][j] = N;
00368 }
00369 }
00370 }
00371 }
00372 if (PyList_Check(base_input_output_list))
00373 {
00374 Py_ssize_t n_inputs_outputs_base = PyList_Size(
base_input_output_list);
00375 self->node_inputs_outputs_base = (Py_ssize_t*)calloc(
n_inputs_outputs_base,sizeof(Py_ssize_t));
00376 assert(self->node_inputs_outputs_base);
00377 for (int i = 0; i < n_inputs_outputs_base; ++i)
00378 {
00379 PyObject *el_i = PyList_GetItem(base_input_output_list,
i);
00380 Py_ssize_t idx = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00381 if (PyErr_Occurred()) return -1;
00382 self->node_inputs_outputs_base[i] = idx;
00383 }
00384 self->node_n_inputs = (Py_ssize_t*)calloc(n_applies,sizeof(
Py_ssize_t));
00385 assert(self->node_n_inputs);
00386 self->node_n_outputs = (Py_ssize_t*)calloc(n_applies,sizeof(
Py_ssize_t));
00387 assert(self->node_n_outputs);
00388 self->node_inputs = (Py_ssize_t**)calloc(n_applies,sizeof(
Py_ssize_t*));
00389 assert(self->node_inputs);
00390 self->node_outputs = (Py_ssize_t**)calloc(n_applies,sizeof(
Py_ssize_t*));
00391 assert(self->node_outputs);
00392 for (int i = 0; i < n_applies; ++i)
00393 {
00394 Py_ssize_t N;
00395 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_inputs, i),
PyExc_IndexError);
00396 if (PyErr_Occurred()) return -1;
00397 assert (N <= n_inputs_outputs_base);
00398 self->node_n_inputs[i] = N;
00399 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_outputs, i
),PyExc_IndexError);
00400 if (PyErr_Occurred()) return -1;
00401 assert (N <= n_inputs_outputs_base);
00402 self->node_n_outputs[i] = N;
00403 N = PyNumber_AsSsize_t(PyList_GetItem(node_input_offset,
i),PyExc_IndexError);
00404 if (PyErr_Occurred()) return -1;
00405 assert (N <= n_inputs_outputs_base);
00406 self->node_inputs[i] = &self->node_inputs_outputs_base[N
];
00407 N = PyNumber_AsSsize_t(PyList_GetItem(node_output_offset
, i),PyExc_IndexError);
00408 if (PyErr_Occurred()) return -1;
00409 assert (N <= n_inputs_outputs_base);
00410 self->node_outputs[i] = &self->node_inputs_outputs_base[
N];
00411 }
00412 }
00413 else
00414 {
00415 PyErr_SetString(PyExc_TypeError, "base_input_output_list
must be list");
00416 return -1;
00417 }
00418
00419 // allocation for var_owner
00420 if (PyList_Check(var_owner))
00421 {
00422 self->var_owner = (Py_ssize_t*)calloc(self->n_vars,sizeof(
Py_ssize_t));
00423 self->var_has_owner = (int*)calloc(self->n_vars,sizeof(int
));
00424 self->var_computed = (int*)calloc(self->n_vars,sizeof(int));
00425 self->var_computed_cells = (PyObject**)calloc(self->n_vars,
sizeof(PyObject*));
00426 self->var_value_cells = (PyObject**)calloc(self->n_vars,
sizeof(PyObject*));
00427 for (int i = 0; i < self->n_vars; ++i)
00428 {
00429 PyObject * el_i = PyList_GetItem(var_owner, i);
00430 if (el_i == Py_None)
00431 {
00432 self->var_has_owner[i] = 0;
00433 }
00434 else
00435 {
00436 Py_ssize_t N = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00437 if (PyErr_Occurred()) return -1;
00438 assert (N <= n_applies);
00439 self->var_owner[i] = N;
00440 self->var_has_owner[i] = 1;
00441 }
00442 self->var_computed_cells[i] = PyList_GetItem(
compute_map_list, i);
00443 Py_INCREF(self->var_computed_cells[i]);
00444 self->var_value_cells[i] = PyList_GetItem(
storage_map_list, i);
00445 Py_INCREF(self->var_value_cells[i]);
00446 }
00447 }
00448 else
00449 {
00450 PyErr_SetString(PyExc_TypeError, "var_owner must be list");
00451 return -1;
00452 }
00453
00454 if (dependencies != Py_None)
00455 {
00456 self->dependencies = (Py_ssize_t**)calloc(self->n_vars,
sizeof(Py_ssize_t *));
00457 self->n_dependencies = (Py_ssize_t*)calloc(self->n_vars,
sizeof(Py_ssize_t));
00458 assert(self->dependencies);
00459 assert(self->n_dependencies);
00460
00461 for (int i = 0; i < self->n_vars; ++i)
00462 {
00463 PyObject *tmp = PyList_GetItem(dependencies, i);
00464 // refcounting - tmp is borrowed
00465 if (unpack_list_of_ssize_t(tmp, &self->dependencies[i],
&self->n_dependencies[i],
00466 "dependencies"))
00467 return -1;
00468 }
00469 }
00470
00471 if (unpack_list_of_ssize_t(output_vars, &self->output_vars, &
self->n_output_vars,
00472 "output_vars"))
00473 return -1;
00474 for (int i = 0; i < self->n_output_vars; ++i)
00475 {
00476 assert(self->output_vars[i] < self->n_vars);
00477 }
00478 if (unpack_list_of_ssize_t(update_storage, &self->update_storage
, &self->n_updates,
00479 "updates_storage"))
00480 return -1;
00481 return 0;
00482 }
00483 static void set_position_of_error(CLazyLinker * self, int owner_idx)
00484 {
00485 if (self->position_of_error == -1)
00486 {
00487 self->position_of_error = owner_idx;
00488 }
00489 }
00490 static PyObject * pycall(CLazyLinker * self, Py_ssize_t node_idx,
int verbose)
00491 {
00492 // call thunk to see which inputs it wants
00493 PyObject * thunk = PyList_GetItem(self->thunks, node_idx);
00494 // refcounting - thunk is borrowed
00495 PyObject * rval = NULL;
00496 if (self->do_timing)
00497 {
00498 double t0 = pytime(NULL);
00499 if (verbose) fprintf(stderr, "calling via Python (node %i)\n",
(int)node_idx);
00500 rval = PyObject_CallObject(thunk, NULL);
00501 if (rval)
00502 {
00503 double t1 = pytime(NULL);
00504 double ti = PyFloat_AsDouble(
00505 PyList_GetItem(self->call_times, node_idx
));
00506 PyList_SetItem(self->call_times, node_idx,
00507 PyFloat_FromDouble(t1 - t0 + ti));
00508 PyObject * count = PyList_GetItem(self->call_counts,
node_idx);
00509 long icount = PyInt_AsLong(count);
00510 PyList_SetItem(self->call_counts, node_idx,
00511 PyInt_FromLong(icount + 1));
00512 }
00513 }
00514 else
00515 {
00516 if (verbose)
00517 {
00518 fprintf(stderr, "calling via Python (node %i)\n", (int)
node_idx);
00519 }
00520 rval = PyObject_CallObject(thunk, NULL);
00521 }
00522 return rval;
00523 }
00524 static int c_call(CLazyLinker * self, Py_ssize_t node_idx, int
verbose)
00525 {
00526 void * ptr_addr = self->thunk_cptr_fn[node_idx];
00527 int (*fn)(void*) = (int (*)(void*))(ptr_addr);
00528 if (verbose) fprintf(stderr, "calling non-lazy shortcut (node
%i)\n", (int)node_idx);
00529 int err = 0;
00530 if (self->do_timing)
00531 {
00532 double t0 = pytime(NULL);
00533 err = fn(self->thunk_cptr_data[node_idx]);
00534 double t1 = pytime(NULL);
00535 double ti = PyFloat_AsDouble(PyList_GetItem(self->call_times,
node_idx));
00536 PyList_SetItem(self->call_times, node_idx, PyFloat_FromDouble(t1
- t0 + ti));
00537 PyObject * count = PyList_GetItem(self->call_counts, node_idx
);
00538 long icount = PyInt_AsLong(count);
00539 PyList_SetItem(self->call_counts, node_idx, PyInt_FromLong(
icount+1));
00540 }
00541 else
00542 {
00543 err = fn(self->thunk_cptr_data[node_idx]);
00544 }
00545
00546 if (err)
00547 {
00548 // cast the argument to a PyList (as described near line 226
of cc.py)
00549 PyObject * __ERROR = ((PyObject**)self->thunk_cptr_data[
node_idx])[0];
00550 assert (PyList_Check(__ERROR));
00551 assert (PyList_Size(__ERROR) == 3);
00552 PyObject * err_type = PyList_GetItem(__ERROR, 0); //stolen ref
00553 PyObject * err_msg = PyList_GetItem(__ERROR, 1); //stolen ref
00554 PyObject * err_trace = PyList_GetItem(__ERROR, 2); //stolen
ref
00555 PyList_SET_ITEM(__ERROR, 0, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00556 PyList_SET_ITEM(__ERROR, 1, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00557 PyList_SET_ITEM(__ERROR, 2, Py_None); Py_INCREF(Py_None); //clobbers
old ref
00558
00559 assert(!PyErr_Occurred()); // because CLinker hid the
exception in __ERROR aka data
00560 PyErr_Restore(err_type, err_msg, err_trace); //steals refs to
args
00561 }
00562 if (err) set_position_of_error(self, node_idx);
00563 return err;
00564 }
00565 static
00566 int lazy_rec_eval(CLazyLinker * self, Py_ssize_t var_idx, PyObject*
one, PyObject*zero)
00567 {
00568 PyObject *rval = NULL;
00569 int verbose = 0;
00570 int err = 0;
00571
00572 if (verbose) fprintf(stderr, "lazy_rec computing %i\n", (int)
var_idx);
00573
00574 if (self->var_computed[var_idx] || !self->var_has_owner[var_idx])
00575 return 0;
00576
00577 Py_ssize_t owner_idx = self->var_owner[var_idx];
00578
00579 // STEP 1: compute the pre-requirements of the node
00580 // Includes input nodes for non-lazy ops.
00581 for (int i = 0; i < self->node_n_prereqs[owner_idx]; ++i)
00582 {
00583 Py_ssize_t prereq_idx = self->node_prereqs[owner_idx][i];
00584 if (!self->var_computed[prereq_idx])
00585 {
00586 err = lazy_rec_eval(self, prereq_idx, one, zero);
00587 if (err) return err;
00588 }
00589 assert (self->var_computed[prereq_idx]);
00590 }
00591
00592 // STEP 2: compute the node itself
00593 if (self->is_lazy[owner_idx])
00594 {
00595 // update the compute_map cells corresponding to the inputs
of this thunk
00596 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00597 {
00598 int in_idx = self->node_inputs[owner_idx][i];
00599 if (self->var_computed[in_idx])
00600 {
00601 Py_INCREF(one);
00602 err = PyList_SetItem(self->var_computed_cells[in_idx],
0, one);
00603 }
00604 else
00605 {
00606 Py_INCREF(zero);
00607 err = PyList_SetItem(self->var_computed_cells[in_idx],
0, zero);
00608 }
00609 if (err) goto fail;
00610 }
00611
00612 rval = pycall(self, owner_idx, verbose);
00613 // refcounting - rval is new ref
00614 //TODO: to prevent infinite loops
00615 // - consider check that a thunk does not ask for an input
that is already computed
00616 if (rval == NULL)
00617 {
00618 assert (PyErr_Occurred());
00619 err = 1;
00620 goto fail;
00621 }
00622
00623 //update the computed-ness of any output cells
00624 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00625 {
00626 int out_idx = self->node_outputs[owner_idx][i];
00627 PyObject * el_i = PyList_GetItem(self->var_computed_cells[
out_idx], 0);
00628 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00629 if (PyErr_Occurred())
00630 {
00631 err = -1;
00632 goto pyfail;
00633 }
00634 assert (N==0 || N==1);
00635 self->var_computed[out_idx] = N;
00636 }
00637 if (!self->var_computed[var_idx])
00638 {
00639 /*
00640 * If self is not computed after the call, this means
that some
00641 * inputs are needed. Compute the ones on the returned
list
00642 * and try to compute the current node again (with
recursive call).
00643 * This allows a node to request more nodes more than
once before
00644 * finally yielding a result.
00645 */
00646 if (!PyList_Check(rval))
00647 {
00648 //TODO: More helpful error to help find *which node*
made this
00649 // bad thunk
00650 PyErr_SetString(PyExc_TypeError,
00651 "lazy thunk should return a list");
00652 err = 1;
00653 goto pyfail;
00654 }
00655
00656 if (!PyList_Size(rval))
00657 {
00658 PyErr_SetString(PyExc_ValueError,
00659 "lazy thunk returned empty list
without computing output");
00660 err = 1;
00661 goto pyfail;
00662 }
00663
00664 for (int i = 0; i < PyList_Size(rval); ++i)
00665 {
00666 PyObject * el_i = PyList_GetItem(rval, i);
00667 Py_ssize_t N = PyNumber_AsSsize_t(el_i,
PyExc_IndexError);
00668 if (PyErr_Occurred())
00669 {
00670 err = 1;
00671 goto pyfail;
00672 }
00673 assert (N <= self->node_n_inputs[owner_idx]);
00674 Py_ssize_t input_idx = self->node_inputs[owner_idx][N
];
00675 err = lazy_rec_eval(self, input_idx, one, zero);
00676 if (err) goto pyfail;
00677 }
00678
00679 Py_DECREF(rval);
00680 /*
00681 * We intentionally skip all the end-of-function
processing
00682 * (mark outputs, GC) as it will be performed by the call
00683 * that actually manages to compute the result.
00684 */
00685 return lazy_rec_eval(self, var_idx, one, zero);
00686 }
00687
00688 Py_DECREF(rval);
00689 }
00690 else //owner is not a lazy op. Ensure all intputs are evaluated.
00691 {
00692 // loop over inputs to owner
00693 // call lazy_rec_eval on each one that is not computed.
00694 // if there's an error, pass it up the stack
00695 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00696 {
00697 Py_ssize_t input_idx = self->node_inputs[owner_idx][i];
00698 if (!self->var_computed[input_idx])
00699 {
00700 err = lazy_rec_eval(self, input_idx, one, zero);
00701 if (err) return err;
00702 }
00703 assert (self->var_computed[input_idx]);
00704 }
00705
00706 // call the thunk for this owner.
00707 if (self->thunk_cptr_fn[owner_idx])
00708 {
00709 err = c_call(self, owner_idx, verbose);
00710 if (err) goto fail;
00711 }
00712 else
00713 {
00714 rval = pycall(self, owner_idx, verbose);
00715 //rval is new ref
00716 if (rval) //pycall returned normally (no exception)
00717 {
00718 if (rval == Py_None)
00719 {
00720 Py_DECREF(rval); //ignore a return of None
00721 }
00722 else if (PyList_Check(rval))
00723 {
00724 PyErr_SetString(PyExc_TypeError,
00725 "non-lazy thunk should return
None, not list");
00726 err = 1;
00727 goto pyfail;
00728 }
00729 else // don't know what it returned, but it wasn't
right.
00730 {
00731 PyErr_SetObject(PyExc_TypeError, rval);
00732 err = 1;
00733 // We don't release rval since we put it in the
error above
00734 goto fail;
00735 }
00736 }
00737 else // pycall returned NULL (internal error)
00738 {
00739 err = 1;
00740 goto fail;
00741 }
00742 }
00743 }
00744
00745 // loop over all outputs and mark them as computed
00746 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00747 {
00748 self->var_computed[self->node_outputs[owner_idx][i]] = 1;
00749 }
00750
00751 // Free vars that are not needed anymore
00752 if (self->allow_gc)
00753 {
00754 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00755 {
00756 int cleanup = 1;
00757 Py_ssize_t i_idx = self->node_inputs[owner_idx][i];
00758 if (!self->var_has_owner[i_idx])
00759 continue;
00760
00761 for (int j = 0; j < self->n_output_vars; ++j)
00762 {
00763 if (i_idx == self->output_vars[j])
00764 {
00765 cleanup = 0;
00766 break;
00767 }
00768 }
00769 if (!cleanup) continue;
00770
00771 for (int j = 0; j < self->n_dependencies[i_idx]; ++j)
00772 {
00773 if (!self->var_computed[self->dependencies[i_idx][j]])
00774 {
00775 cleanup = 0;
00776 break;
00777 }
00778 }
00779 if (!cleanup) continue;
00780
00781 Py_INCREF(Py_None);
00782 err = PyList_SetItem(self->var_value_cells[i_idx], 0,
Py_None);
00783 //See the Stack gc implementation for why we change it to 2 and not
0.
00784 self->var_computed[i_idx] = 2;
00785 if (err) goto fail;
00786 }
00787 }
00788
00789 return 0;
00790 pyfail:
00791 Py_DECREF(rval);
00792 fail:
00793 set_position_of_error(self, owner_idx);
00794 return err;
00795 }
00796
00797 static PyObject *
00798 CLazyLinker_call(PyObject *_self, PyObject *args, PyObject *kwds)
00799 {
00800 CLazyLinker * self = (CLazyLinker*)_self;
00801 static char *kwlist[] = {
00802 (char*)"time_thunks",
00803 (char *)"n_calls",
00804 NULL};
00805 int n_calls=1;
00806 if (! PyArg_ParseTupleAndKeywords(args, kwds, "|ii", kwlist,
00807 &self->do_timing,
00808 &n_calls))
00809 return NULL;
00810 int err = 0;
00811 self->position_of_error = -1;
00812 // create constants used to fill the var_compute_cells
00813 PyObject * one = PyInt_FromLong(1);
00814 PyObject * zero = PyInt_FromLong(0);
00815
00816 // pre-allocate our return value
00817 Py_INCREF(Py_None);
00818 PyObject * rval = Py_None;
00819 //clear storage of pre_call_clear elements
00820 for (int call_i = 0; call_i < n_calls && (!err); ++call_i)
00821 {
00822 Py_ssize_t n_pre_call_clear = PyList_Size(self->pre_call_clear
);
00823 assert(PyList_Check(self->pre_call_clear));
00824 for (int i = 0; i < n_pre_call_clear; ++i)
00825 {
00826 PyObject * el_i = PyList_GetItem(self->pre_call_clear, i);
00827 Py_INCREF(Py_None);
00828 PyList_SetItem(el_i, 0, Py_None);
00829 }
00830 //clear the computed flag out of all non-input vars
00831 for (int i = 0; i < self->n_vars; ++i)
00832 {
00833 self->var_computed[i] = !self->var_has_owner[i];
00834 if (self->var_computed[i])
00835 {
00836 Py_INCREF(one);
00837 PyList_SetItem(self->var_computed_cells[i], 0, one);
00838 }
00839 else
00840 {
00841 Py_INCREF(zero);
00842 PyList_SetItem(self->var_computed_cells[i], 0, zero);
00843 }
00844 }
00845
00846 for (int i = 0; i < self->n_output_vars && (!err); ++i)
00847 {
00848 err = lazy_rec_eval(self, self->output_vars[i], one, zero
);
00849 }
00850
00851 if (!err)
00852 {
00853 // save references to outputs prior to updating storage
containers
00854 assert (self->n_output_vars >= self->n_updates);
00855 Py_DECREF(rval);
00856 rval = PyList_New(self->n_output_vars);
00857 for (int i = 0; i < (self->n_output_vars); ++i)
00858 {
00859 Py_ssize_t src = self->output_vars[i];
00860 PyObject * item = PyList_GetItem(self->var_value_cells
[src], 0);
00861 if (self->var_computed[src] != 1)
00862 {
00863 err = 1;
00864 PyErr_Format(PyExc_AssertionError,
00865 "The compute map of output %d should
contain "
00866 "1 at the end of execution, not %d.",
00867 i, self->var_computed[src]);
00868 break;
00869 }
00870 Py_INCREF(item);
00871 PyList_SetItem(rval, i, item);
00872 }
00873 }
00874
00875 if (!err)
00876 {
00877 // Update the inputs that have an update rule
00878 for (int i = 0; i < self->n_updates; ++i)
00879 {
00880 PyObject* tmp = PyList_GetItem(rval, self->n_output_vars
- self->n_updates + i);
00881 Py_INCREF(tmp);
00882 Py_ssize_t dst = self->update_storage[i];
00883 PyList_SetItem(self->var_value_cells[dst], 0, tmp);
00884 }
00885 }
00886 }
00887
00888 /*
00889 Clear everything that is left and not an output. This is needed
00890 for lazy evaluation since the current GC algo is too
conservative
00891 with lazy graphs.
00892 */
00893 if (self->allow_gc && !err)
00894 {
00895 for (Py_ssize_t i = 0; i < self->n_vars; ++i)
00896 {
00897 int do_cleanup = 1;
00898 if (!self->var_has_owner[i] || !self->var_computed[i])
00899 continue;
00900 for (int j = 0; j < self->n_output_vars; ++j)
00901 {
00902 if (i == self->output_vars[j])
00903 {
00904 do_cleanup = 0;
00905 break;
00906 }
00907 }
00908 if (!do_cleanup)
00909 continue;
00910 Py_INCREF(Py_None);
00911 PyList_SetItem(self->var_value_cells[i], 0, Py_None);
00912 }
00913 }
00914 Py_DECREF(one);
00915 Py_DECREF(zero);
00916 if (err)
00917 {
00918 Py_DECREF(rval);
00919 return NULL;
00920 }
00921 return rval;
00922 }
00923
00924 #if 0
00925 static PyMethodDef CLazyLinker_methods[] = {
00926 {
00927 //"name", (PyCFunction)CLazyLinker_accept, METH_VARARGS,
"Return the name, combining the first and last name"
00928 },
00929 {NULL} /* Sentinel */
00930 };
00931 #endif
00932
00933
00934 static PyObject *
00935 CLazyLinker_get_allow_gc(CLazyLinker *self, void *closure)
00936 {
00937 return PyBool_FromLong(self->allow_gc);
00938 }
00939
00940 static int
00941 CLazyLinker_set_allow_gc(CLazyLinker *self, PyObject *value, void *
closure)
00942 {
00943 if(!PyBool_Check(value))
00944 return -1;
00945
00946 if (value == Py_True)
00947 self->allow_gc = true;
00948 else
00949 self->allow_gc = false;
00950 return 0;
00951 }
00952
00953 static PyGetSetDef CLazyLinker_getset[] = {
00954 {(char*)"allow_gc",
00955 (getter)CLazyLinker_get_allow_gc,
00956 (setter)CLazyLinker_set_allow_gc,
00957 (char*)"do this function support allow_gc",
00958 NULL},
00959 {NULL, NULL, NULL, NULL} /* Sentinel */
00960 };
00961 static PyMemberDef CLazyLinker_members[] = {
00962 {(char*)"nodes", T_OBJECT_EX, offsetof(CLazyLinker, nodes), 0,
00963 (char*)"list of nodes"},
00964 {(char*)"thunks", T_OBJECT_EX, offsetof(CLazyLinker, thunks), 0,
00965 (char*)"list of thunks in program"},
00966 {(char*)"call_counts", T_OBJECT_EX, offsetof(CLazyLinker,
call_counts), 0,
00967 (char*)"number of calls of each thunk"},
00968 {(char*)"call_times", T_OBJECT_EX, offsetof(CLazyLinker,
call_times), 0,
00969 (char*)"total runtime in each thunk"},
00970 {(char*)"position_of_error", T_INT, offsetof(CLazyLinker,
position_of_error), 0,
00971 (char*)"position of failed thunk"},
00972 {(char*)"time_thunks", T_INT, offsetof(CLazyLinker, do_timing),
0,
00973 (char*)"bool: nonzero means call will time thunks"},
00974 {(char*)"need_update_inputs", T_INT, offsetof(CLazyLinker,
need_update_inputs), 0,
00975 (char*)"bool: nonzero means Function.__call__ must implement
update mechanism"},
00976 {NULL} /* Sentinel */
00977 };
00978
00979 static PyTypeObject lazylinker_ext_CLazyLinkerType = {
00980 #if defined(NPY_PY3K)
00981 PyVarObject_HEAD_INIT(NULL, 0)
00982 #else
00983 PyObject_HEAD_INIT(NULL)
00984 0, /*ob_size*/
00985 #endif
00986 "lazylinker_ext.CLazyLinker", /*tp_name*/
00987 sizeof(CLazyLinker), /*tp_basicsize*/
00988 0, /*tp_itemsize*/
00989 CLazyLinker_dealloc, /*tp_dealloc*/
00990 0, /*tp_print*/
00991 0, /*tp_getattr*/
00992 0, /*tp_setattr*/
00993 0, /*tp_compare*/
00994 0, /*tp_repr*/
00995 0, /*tp_as_number*/
00996 0, /*tp_as_sequence*/
00997 0, /*tp_as_mapping*/
00998 0, /*tp_hash */
00999 CLazyLinker_call, /*tp_call*/
01000 0, /*tp_str*/
01001 0, /*tp_getattro*/
01002 0, /*tp_setattro*/
01003 0, /*tp_as_buffer*/
01004 Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
01005 "CLazyLinker object", /* tp_doc */
01006 0, /* tp_traverse */
01007 0, /* tp_clear */
01008 0, /* tp_richcompare */
01009 0, /* tp_weaklistoffset */
01010 0, /* tp_iter */
01011 0, /* tp_iternext */
01012 0,//CLazyLinker_methods, /* tp_methods */
01013 CLazyLinker_members, /* tp_members */
01014 CLazyLinker_getset, /* tp_getset */
01015 0, /* tp_base */
01016 0, /* tp_dict */
01017 0, /* tp_descr_get */
01018 0, /* tp_descr_set */
01019 0, /* tp_dictoffset */
01020 (initproc)CLazyLinker_init,/* tp_init */
01021 0, /* tp_alloc */
01022 CLazyLinker_new, /* tp_new */
01023 };
01024
01025 static PyObject * get_version(PyObject *dummy, PyObject *args)
01026 {
01027 PyObject *result = PyFloat_FromDouble(0.21);
01028 return result;
01029 }
01030
01031 static PyMethodDef lazylinker_ext_methods[] = {
01032 {"get_version", get_version, METH_VARARGS, "Get extension
version."},
01033 {NULL, NULL, 0, NULL} /* Sentinel */
01034 };
01035
01036 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
01037 #define PyMODINIT_FUNC void
01038 #endif
01039
01040 #if defined(NPY_PY3K)
01041 static struct PyModuleDef moduledef = {
01042 PyModuleDef_HEAD_INIT,
01043 "lazylinker_ext",
01044 NULL,
01045 -1,
01046 lazylinker_ext_methods,
01047 NULL,
01048 NULL,
01049 NULL,
01050 NULL
01051 };
01052 #endif
01053 #if defined(NPY_PY3K)
01054 #define RETVAL m
01055 PyMODINIT_FUNC
01056 PyInit_lazylinker_ext(void) {
01057 #else
01058 #define RETVAL
01059 PyMODINIT_FUNC
01060 initlazylinker_ext(void)
01061 {
01062 #endif
01063 PyObject* m;
01064
01065 lazylinker_ext_CLazyLinkerType.tp_new = PyType_GenericNew;
01066 if (PyType_Ready(&lazylinker_ext_CLazyLinkerType) < 0)
01067 return RETVAL;
01068 #if defined(NPY_PY3K)
01069 m = PyModule_Create(&moduledef);
01070 #else
01071 m = Py_InitModule3("lazylinker_ext", lazylinker_ext_methods,
01072 "Example module that creates an extension
type.");
01073 #endif
01074 Py_INCREF(&lazylinker_ext_CLazyLinkerType);
01075 PyModule_AddObject(m, "CLazyLinker", (PyObject *)&
lazylinker_ext_CLazyLinkerType);
01076
01077 return RETVAL;
01078 }
01079
01080
===============================
Problem occurred during compilation with the command line below:
/usr/bin/g++ -shared -g -march=core2 -mcx16 -msahf -mpopcnt -msse4.2 --param
l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=256 -
mtune=core2 -D NPY_NO_DEPRECATED_API=NPY_1_7
_API_VERSION -m64 -fPIC -I/app/.heroku/python/lib/python2.7/site-packages/
numpy/core/include -I/app/.heroku/python/include/python2.7 -o /app/.theano/
compiledir_Linux-3.13--generic-x86_64-with-deb
ian-squeeze-sid--2.7.8-64/lazylinker_ext/lazylinker_ext.so /app/.theano/
compiledir_Linux-3.13--generic-x86_64-with-debian-squeeze-sid--2.7.8-64/
lazylinker_ext/mod.cpp -L/app/.heroku/python/lib -l
python2.7
/usr/bin/ld: /app/.heroku/python/lib/libpython2.7.a(abstract.o): relocation
R_X86_64_32 against `.rodata.str1.8' can not be used when making a shared
object; recompile with -fPIC
/app/.heroku/python/lib/libpython2.7.a: could not read symbols: Bad value
collect2: ld returned 1 exit status
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/app/src/theano/theano/__init__.py", line 55, in <module>
from theano.compile import \
File "/app/src/theano/theano/compile/__init__.py", line 9, in <module>
from theano.compile.function_module import *
File "/app/src/theano/theano/compile/function_module.py", line 18, in
<module>
import theano.compile.mode
File "/app/src/theano/theano/compile/mode.py", line 11, in <module>
import theano.gof.vm
File "/app/src/theano/theano/gof/vm.py", line 568, in <module>
import lazylinker_c
File "/app/src/theano/theano/gof/lazylinker_c.py", line 116, in <module>
preargs=args)
File "/app/src/theano/theano/gof/cmodule.py", line 1952, in compile_str
(status, compile_stderr.replace('\n', '. ')))
Exception: Compilation failed (return status=1): /usr/bin/ld:
/app/.heroku/python/lib/libpython2.7.a(abstract.o): relocation R_X86_64_32
against `.rodata.str1.8' can not be used when making a sha
red object; recompile with -fPIC. /app/.heroku/python/lib/libpython2.7.a:
could not read symbols: Bad value. collect2: ld returned 1 exit status.
--
---
You received this message because you are subscribed to the Google Groups "theano-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to theano-users+***@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
---
You received this message because you are subscribed to the Google Groups "theano-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to theano-users+***@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.