It looks like one of the updates is being upcast to float64. Can you explicitly cast them to float32?
On Thursday, July 28, 2016 at 1:50:32 PM UTC-7, Steve Leach wrote: > > I posted this in lasagne-users as well: Running into a strange error when > trying to calculate updates. The end > goal is a learned pruning/dropping layer, who's total number of > activations along with how binary those > activations are are part of the cost function - these activations from its > own weights are then multiplied by the > incoming layer in order to selectively turn off the least useful inputs. > Once the mask is learned, the layers > before and after can be resized. Seems sound in principle, but I'm having > trouble convincing theano of the idea. > > > The code: > > ------------------------------------------------------------------------------------------------------------------------------------------ > from lasagne.layers import InputLayer, DenseLayer, Layer, get_output > from lasagne.nonlinearities import sigmoid, rectify, tanh > from numpy.random import random > from numpy.random import randint > from numpy import zeros > from theano import shared > import theano as th > import theano.tensor as T > import numpy as np > import lasagne as nn > floatX = th.config.floatX > batchsize = 4 > Width = 10 > Target = 5 > np.set_printoptions(precision=2, suppress=True) > > class Mask(Layer): > def __init__(self, incoming, A=10**4, **kwargs): > super(Mask, self).__init__(incoming, **kwargs) > num_inputs = self.input_shape[1] > self.A = A > self.num_units = num_inputs > W = ((random( (batchsize,num_inputs))-(1/2))*.001).astype(floatX) > for n in range(1,W.shape[0]): > W[n]=W[0] > W = shared(W ) > self.W = self.add_param(W,(batchsize,num_inputs), name = 'W_Mask') > > def get_output_for(self, input, **kwargs): > #return ((sigmoid(rectify(self.W*self.A) )-(1/2))*2) > return ((tanh(self.A*self.W)/2)+.5 +(input*0) ).astype(floatX) > > def average(self): > Acc = zeros((1,self.num_units)) > W = self.W.get_value() > for w in W: > Acc += w > Acc /= batchsize > W = zeros((batchsize,self.num_units)) > for n in range(batchsize): > W[n] = Acc[:] > self.W.set_value(W) > > class MaskLayer(Layer): > def __init__(self, incoming, W, A=10**6, **kwargs): > super(MaskLayer, self).__init__(incoming, **kwargs) > num_inputs = self.input_shape[1] > self.A = A > self.num_units = num_inputs > self.W = self.add_param(W,(1,num_inputs), name = 'W') > def get_output_for(self, input, **kwargs): > return (((tanh(self.A*self.W)/2)+.5)*input).astype(floatX) > > > In = InputLayer((3,Width),input_var=x) > M = Mask(In) > LM = MaskLayer(In,M.W) > LN = DenseLayer(LM,1,nonlinearity=sigmoid) > X = (random((batchsize,Width))-.5).astype(floatX) > x = T.matrix('x') > Out = get_output(LN,x) > M1 = get_output(M,x) > params = nn.layers.get_all_params(LN) > > mask = th.function([x],M1) > > cost = ((Target - (M1.sum().mean() / batchsize))**2 + > 1000*T.sin(M1*np.pi).mean() - Out.mean()).astype(floatX) > Updates = nn.updates.sgd(cost,params,learning_rate=.001) > #train = th.function([x],[cost],updates=Updates) > run = th.function([x],[cost, Out, M1],on_unused_input='ignore', > updates=Updates) > print('In:\n',X) > print('Mask:\n',M.get_output_for(X).eval()) > print('Masked:\n',LM.get_output_for(X).eval()) > print('Dense:\n',LN.get_output_for(X).eval()) > > > The Error: > > -------------------------------------------------------------------------------------------------------- > > ---------------------------------------------------------------------------TypeError > Traceback (most recent call > last)/usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in > rebuild_collect_shared(outputs, inputs, replace, updates, rebuild_strict, > copy_inputs_over, no_default_updates) 192 update_val = > store_into.type.filter_variable(update_val,--> 193 > allow_convert=False) 194 except > TypeError: > > /usr/local/lib/python3.4/dist-packages/theano/tensor/type.py in > filter_variable(self, other, allow_convert) 233 > other=other,--> 234 self=self)) 235 > TypeError: Cannot convert Type TensorType(float64, matrix) (of Variable > Elemwise{sub,no_inplace}.0) into Type TensorType(float32, matrix). You can > try to manually convert Elemwise{sub,no_inplace}.0 into a TensorType(float32, > matrix). > > During handling of the above exception, another exception occurred: > TypeError Traceback (most recent call last) > <ipython-input-70-60c11801fab9> in <module>() 68 Updates = > nn.updates.sgd(cost,params,learning_rate=.001) 69 #train = > th.function([x],[cost],updates=Updates)---> 70 run = th.function([x],[cost, > Out, M1],on_unused_input='ignore', updates=Updates) > 71 print('In:\n',X) > 72 print('Mask:\n',M.get_output_for(X).eval()) > > /usr/local/lib/python3.4/dist-packages/theano/compile/function.py in > function(inputs, outputs, mode, updates, givens, no_default_updates, > accept_inplace, name, rebuild_strict, allow_input_downcast, profile, > on_unused_input) 320 on_unused_input=on_unused_input, > 321 profile=profile,--> 322 > output_keys=output_keys) 323 # We need to add the flag check_aliased > inputs if we have any mutable or > 324 # borrowed used defined inputs > > /usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in > pfunc(params, outputs, mode, updates, givens, no_default_updates, > accept_inplace, name, rebuild_strict, allow_input_downcast, profile, > on_unused_input, output_keys) 441 > rebuild_strict=rebuild_strict, 442 > copy_inputs_over=True,--> 443 > no_default_updates=no_default_updates) 444 # extracting the arguments > 445 input_variables, cloned_extended_outputs, other_stuff = > output_vars > > /usr/local/lib/python3.4/dist-packages/theano/compile/pfunc.py in > rebuild_collect_shared(outputs, inputs, replace, updates, rebuild_strict, > copy_inputs_over, no_default_updates) 206 ' > function to remove broadcastable dimensions.') 207 --> 208 > raise TypeError(err_msg, err_sug) 209 assert update_val.type == > store_into.type 210 > > TypeError: ('An update must have the same type as the original shared > variable (shared_var=W_Mask, shared_var.type=TensorType(float32, matrix), > update_val=Elemwise{sub,no_inplace}.0, update_val.type=TensorType(float64, > matrix)).', 'If the difference is related to the broadcast pattern, you can > call the tensor.unbroadcast(var, axis_to_unbroadcast[, ...]) function to > remove broadcastable dimensions.') > > -- --- You received this message because you are subscribed to the Google Groups "theano-users" group. To unsubscribe from this group and stop receiving emails from it, send an email to theano-users+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.