pythontheanobayesianpymc3

PyMC3 Minibatch ADVI


I’m trying to use PyMC3 Minibatch ADVI for Bayesian Regression. The pm.fit function throws the following error and I’m not sure how to fix it.

It says that the ‘str’ object has no attribute ‘type’. What is any ‘str’ object from the error message here? I’ve mapped float tensors for more_replacements to the best of what I know.

advi = pm.ADVI()
tracker = pm.callbacks.Tracker(mean=advi.approx.mean.eval,std=advi.approx.std.eval)
map_tensor_batch = {'x_tensor': pm.Minibatch(X_train, dtype=float),'y_tensor':pm.Minibatch(y_train['target'],dtype=float)}
approx = advi.fit(20000, obj_optimizer=pm.sgd(learning_rate=0.01), callbacks=[tracker], more_replacements = map_tensor_batch)

Your answers will be appreciated.

Addendum: If I just say

pm.Minibatch(np.array([tuple(y.iloc[i,[0]]) for i in train_index])).type()

(Or)

map_tensor_batch['y_tensor'].type()

I get the following result:

<TensorType(float64, matrix)>

Then why does it throw the attribute error below? Again, what’s my ‘str’ object?

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-24-2824def803dc> in <module>
     56         tracker = pm.callbacks.Tracker(mean=advi.approx.mean.eval,std=advi.approx.std.eval)
     57         map_tensor_batch = {'x_tensor': pm.Minibatch(X_train, dtype=float),'y_tensor': pm.Minibatch(y_train['target'],dtype=float)}
---> 58         approx = advi.fit(20000, obj_optimizer=pm.sgd(learning_rate=0.01), callbacks=[tracker], more_replacements = map_tensor_batch)
     59         fig = plt.figure(figsize=(16, 9))
     60         mu_ax = fig.add_subplot(221)

/opt/conda/lib/python3.7/site-packages/pymc3/variational/inference.py in fit(self, n, score, callbacks, progressbar, **kwargs)
    142             callbacks = []
    143         score = self._maybe_score(score)
--> 144         step_func = self.objective.step_function(score=score, **kwargs)
    145         if progressbar:
    146             progress = progress_bar(range(n), display=progressbar)

/opt/conda/lib/python3.7/site-packages/theano/configparser.py in res(*args, **kwargs)
     46         def res(*args, **kwargs):
     47             with self:
---> 48                 return f(*args, **kwargs)
     49 
     50         return res

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in step_function(self, obj_n_mc, tf_n_mc, obj_optimizer, test_optimizer, more_obj_params, more_tf_params, more_updates, more_replacements, total_grad_norm_constraint, score, fn_kwargs)
    358             more_updates=more_updates,
    359             more_replacements=more_replacements,
--> 360             total_grad_norm_constraint=total_grad_norm_constraint,
    361         )
    362         if score:

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in updates(self, obj_n_mc, tf_n_mc, obj_optimizer, test_optimizer, more_obj_params, more_tf_params, more_updates, more_replacements, total_grad_norm_constraint)
    244             more_obj_params=more_obj_params,
    245             more_replacements=more_replacements,
--> 246             total_grad_norm_constraint=total_grad_norm_constraint,
    247         )
    248         resulting_updates.update(more_updates)

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in add_obj_updates(self, updates, obj_n_mc, obj_optimizer, more_obj_params, more_replacements, total_grad_norm_constraint)
    284             more_replacements = dict()
    285         obj_target = self(
--> 286             obj_n_mc, more_obj_params=more_obj_params, more_replacements=more_replacements
    287         )
    288         grads = pm.updates.get_or_compute_grads(obj_target, self.obj_params + more_obj_params)

/opt/conda/lib/python3.7/site-packages/theano/configparser.py in res(*args, **kwargs)
     46         def res(*args, **kwargs):
     47             with self:
---> 48                 return f(*args, **kwargs)
     49 
     50         return res

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in __call__(self, nmc, **kwargs)
    401             m = 1.0
    402         a = self.op.apply(self.tf)
--> 403         a = self.approx.set_size_and_deterministic(a, nmc, 0, kwargs.get("more_replacements"))
    404         return m * self.op.T(a)
    405 

/opt/conda/lib/python3.7/site-packages/theano/configparser.py in res(*args, **kwargs)
     46         def res(*args, **kwargs):
     47             with self:
---> 48                 return f(*args, **kwargs)
     49 
     50         return res

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in set_size_and_deterministic(self, node, s, d, more_replacements)
   1496         _node = node
   1497         optimizations = self.get_optimization_replacements(s, d)
-> 1498         flat2rand = self.make_size_and_deterministic_replacements(s, d, more_replacements)
   1499         node = theano.clone(node, optimizations)
   1500         node = theano.clone(node, flat2rand)

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in make_size_and_deterministic_replacements(self, s, d, more_replacements)
   1470         flat2rand = collections.OrderedDict()
   1471         for g in self.groups:
-> 1472             flat2rand.update(g.make_size_and_deterministic_replacements(s, d, more_replacements))
   1473         flat2rand.update(more_replacements)
   1474         return flat2rand

/opt/conda/lib/python3.7/site-packages/pymc3/variational/opvi.py in make_size_and_deterministic_replacements(self, s, d, more_replacements)
   1186         initial = tt.patternbroadcast(initial, self.symbolic_initial.broadcastable)
   1187         if more_replacements:
-> 1188             initial = theano.clone(initial, more_replacements)
   1189         return {self.symbolic_initial: initial}
   1190 

/opt/conda/lib/python3.7/site-packages/theano/scan/utils.py in clone(output, replace, strict, share_inputs)
    212             )
    213         )
--> 214     tmp_replace = [(x, x.type()) for x, y in items]
    215     new_replace = [(x, y) for ((_, x), (_, y)) in zip(tmp_replace, items)]
    216     _, _outs, _ = rebuild_collect_shared(

/opt/conda/lib/python3.7/site-packages/theano/scan/utils.py in <listcomp>(.0)
    212             )
    213         )
--> 214     tmp_replace = [(x, x.type()) for x, y in items]
    215     new_replace = [(x, y) for ((_, x), (_, y)) in zip(tmp_replace, items)]
    216     _, _outs, _ = rebuild_collect_shared(

AttributeError: 'str' object has no attribute 'type'

A workaround seems to be to use a tuple instead of a dictionary. The following doesn’t throw the error as before.

advi = pm.ADVI()
tracker = pm.callbacks.Tracker(mean=advi.approx.mean.eval,std=advi.approx.std.eval)
#map_tensor_batch = {'x_tensor': pm.Minibatch(x_tensor.eval()),'y_tensor': pm.Minibatch(y_tensor.eval())}  
map_tensor_batch = (pm.Minibatch(x_tensor.eval()),pm.Minibatch(y_tensor.eval()))
approx = advi.fit(20000, obj_optimizer=pm.sgd(learning_rate=0.00001), callbacks=[tracker], more_replacements = map_tensor_batch)
    

But there’s another error as below:

TypeError: TensorType does not support iteration. Maybe you are using builtins.sum instead of theano.tensor.sum? (Maybe .max?)

Any fix to this?

This (https://alexioannides.com/2018/11/07/bayesian-regression-in-pymc3-using-mcmc-variational-inference/) is the example I’m trying to follow.


Solution

  • The blog post you are working from shows

    import theano
    
    y_tensor = theano.shared(train.y.values.astype('float64'))
    x_tensor = theano.shared(train.x.values.astype('float64'))
    
    map_tensor_batch = {y_tensor: pm.Minibatch(train.y.values, 100),
                        x_tensor: pm.Minibatch(train.x.values, 100)}
    

    That is, map_tensor_batch should be a dict, but the keys are Theano tensors, not mere strings.