我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用lib.floatX()。
def create_model(inp): out = (inp.astype(theano.config.floatX)/lib.floatX(Q_LEVELS-1) - lib.floatX(0.5)) l_out = out.dimshuffle(0,1,'x') skips = [] for i in range(args.wavenet_blocks): l_out, skip_out = create_wavenet_block(l_out, args.dilation_layers_per_block, 1 if i == 0 else args.dim, args.dim, name = "block_{}".format(i+1)) skips.append(skip_out) out = skips[-1] for i in range(args.wavenet_blocks - 1): out = out + skips[args.wavenet_blocks - 2 - i][:,(2**args.dilation_layers_per_block - 1)*(i+1):] for i in range(3): out = lib.ops.conv1d("out_{}".format(i+1), out, args.dim, args.dim, 1, non_linearity='relu') out = lib.ops.conv1d("final", out, args.dim, args.q_levels, 1, non_linearity='identity') return out
def gaussian_nll(x, mus, sigmas): """ NLL for Multivariate Normal with diagonal covariance matrix See: wikipedia.org/wiki/Multivariate_normal_distribution#Likelihood_function where \Sigma = diag(s_1^2,..., s_n^2). x, mus, sigmas all should have the same shape. sigmas (s_1,..., s_n) should be strictly positive. Results in output shape of similar but without the last dimension. """ nll = lib.floatX(numpy.log(2. * numpy.pi)) nll += 2. * T.log(sigmas) nll += ((x - mus) / sigmas) ** 2. nll = nll.sum(axis=-1) nll *= lib.floatX(0.5) return nll
def recurrent_fn_hred(x_t, h_tm1,hidden_dim,W1,b1,W2,b2): global DIM #A1 = T.nnet.sigmoid(lib.ops.BatchNorm(T.dot(T.concatenate((x_t,h_tm1),axis=1),W1),name="FrameLevel.GRU"+str(name)+".Input.",length=2*512) + b1) A1 = T.nnet.sigmoid(T.dot(T.concatenate((x_t,h_tm1),axis=1),W1) + b1) z = A1[:,:hidden_dim] r = A1[:,hidden_dim:] scaled_hidden = r*h_tm1 #h = T.tanh(lib.ops.BatchNorm(T.dot(T.concatenate((scaled_hidden,x_t),axis=1),W2),name="FrameLevel.GRU"+str(name)+".Output.",length=512)+b2) h = T.tanh(T.dot(T.concatenate((scaled_hidden,x_t),axis=1),W2) + b2) one = lib.floatX(1.0) return ((z * h) + ((one - z) * h_tm1)).astype('float32')
def Adam(cost, params, lr=0.01, beta1=0.9, beta2=0.999, epsilon=1e-8,gradClip=True,value=1.): gparams = [] iter = 1 for param in params: gparam = T.grad(cost,param) if gradClip: gparam = T.clip(gparam,lib.floatX(-value), lib.floatX(value)) gparams.append(gparam) print str(iter) + " completed" iter += 1 updates = [] for p, g in zip(params, gparams): m = theano.shared(p.get_value() * 0.) v = theano.shared(p.get_value() * 0.) m_new = beta1 * m + (1 - beta1) * g v_new = beta2 * v + (1 - beta2) * (g ** 2) gradient_scaling = T.sqrt(v_new + epsilon) updates.append((m, m_new)) updates.append((v, v_new)) updates.append((p, p - lr * m / gradient_scaling)) return updates
def DiagonalBiLSTM(name, input_dim, inputs): """ inputs.shape: (batch size, height, width, input_dim) inputs.shape: (batch size, height, width, DIM) """ forward = DiagonalLSTM(name+'.Forward', input_dim, inputs) backward = DiagonalLSTM(name+'.Backward', input_dim, inputs[:,:,::-1,:])[:,:,::-1,:] batch_size = inputs.shape[0] backward = T.concatenate([ T.zeros([batch_size, 1, WIDTH, DIM], dtype=theano.config.floatX), backward[:, :-1, :, :] ], axis=1) return forward + backward # inputs.shape: (batch size, height, width, channels)
def create_wavenet_block(inp, num_dilation_layer, input_dim, output_dim, name =None): assert name is not None layer_out = inp skip_contrib = [] skip_weights = lib.param(name+".parametrized_weights", lib.floatX(numpy.ones((num_dilation_layer,)))) for i in range(num_dilation_layer): layer_out, skip_c = lib.ops.dil_conv_1D( layer_out, output_dim, input_dim if i == 0 else output_dim, 2, dilation = 2**i, non_linearity = 'gated', name = name+".dilation_{}".format(i+1) ) skip_c = skip_c*skip_weights[i] skip_contrib.append(skip_c) skip_out = skip_contrib[-1] j = 0 for i in range(num_dilation_layer-1): j += 2**(num_dilation_layer-i-1) skip_out = skip_out + skip_contrib[num_dilation_layer-2 - i][:,j:] return layer_out, skip_out
def uniform(stdev, size): """ uniform distribution with the given stdev and size From Ishaan's code: https://github.com/igul222/speech """ return numpy.random.uniform( low=-stdev * numpy.sqrt(3), high=stdev * numpy.sqrt(3), size=size ).astype(theano.config.floatX)
def Embedding(name, n_symbols, output_dim, indices): vectors = lib.param( name, numpy.random.randn( n_symbols, output_dim ).astype(theano.config.floatX) ) output_shape = [ indices.shape[i] for i in xrange(indices.ndim) ] + [output_dim] return vectors[indices.flatten()].reshape(output_shape)
def softmax_and_sample(logits): old_shape = logits.shape flattened_logits = logits.reshape((-1, logits.shape[logits.ndim-1])) samples = T.cast( srng.multinomial(pvals=T.nnet.softmax(flattened_logits)), theano.config.floatX ).reshape(old_shape) return T.argmax(samples, axis=samples.ndim-1) # TODO: Have a look at this benchmark: # https://github.com/MaximumEntropy/cudnn_rnn_theano_benchmarks
def GMM_nll(x, mus, sigmas, mix_weights): """ D is dimension of each observation (e.g. frame_size) for each component (multivariate Normal with diagonal covariance matrix) See `gaussian_nll` x : (batch_size, D) mus : (batch_size, D, num_gaussians) sigmas : (batch_size, D, num_gaussians) mix_weights : (batch_size, num_gaussians) """ x = x.dimshuffle(0, 1, 'x') # Similar to `gaussian_nll` ll_component_wise = lib.floatX(numpy.log(2. * numpy.pi)) ll_component_wise += 2. * T.log(sigmas) ll_component_wise += ((x - mus) / sigmas) ** 2. ll_component_wise = ll_component_wise.sum(axis=1) # on FRAME_SIZE ll_component_wise *= lib.floatX(-0.5) # LL not NLL # Now ready to take care of weights of each component # Simply applying exp could potentially cause inf/NaN. # Look up LogSumExp trick, Softmax in theano, or this: # hips.seas.harvard.edu/blog/2013/01/09/computing-log-sum-exp/ weighted_ll = ll_component_wise + T.log(mix_weights) ll_max = T.max(weighted_ll, axis=1, keepdims=True) nll = T.log(T.sum(T.exp(weighted_ll - ll_max), axis=1, keepdims=True)) nll += ll_max nll = -nll.sum(axis=1) return nll
def T_one_hot(inp_tensor, n_classes): """ :todo: - Implement other methods from here: - Compare them speed-wise for different sizes - Implement N_one_hot for Numpy version, with speed tests. Theano one-hot (1-of-k) from an input tensor of indecies. If the indecies are of the shape (a0, a1, ..., an) the output shape would be (a0, a1, ..., a2, n_classes). :params: - inp_tensor: any theano tensor with dtype int* as indecies and all of them between [0, n_classes-1]. - n_classes: number of classes which determines the output size. :usage: >>> idx = T.itensor3() >>> idx_val = numpy.array([[[0,1,2,3],[4,5,6,7]]], dtype='int32') >>> one_hot = T_one_hot(t, 8) >>> one_hot.eval({idx:idx_val}) >>> print out array([[[[ 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0.], [ 0., 0., 0., 1., 0., 0., 0., 0.]], [[ 0., 0., 0., 0., 1., 0., 0., 0.], [ 0., 0., 0., 0., 0., 1., 0., 0.], [ 0., 0., 0., 0., 0., 0., 1., 0.], [ 0., 0., 0., 0., 0., 0., 0., 1.]]]]) >>> print idx_val.shape, out.shape (1, 2, 4) (1, 2, 4, 8) """ flattened = inp_tensor.flatten() z = T.zeros((flattened.shape[0], n_classes), dtype=theano.config.floatX) one_hot = T.set_subtensor(z[T.arange(flattened.shape[0]), flattened], 1) out_shape = [inp_tensor.shape[i] for i in xrange(inp_tensor.ndim)] + [n_classes] one_hot = one_hot.reshape(out_shape) return one_hot
def FrameProcessor(frames): """ frames.shape: (batch size, n frames, FRAME_SIZE) output.shape: (batch size, n frames, DIM) """ embedded = lib.ops.Embedding('FrameEmbedding', Q_LEVELS, Q_LEVELS, frames) embedded = embedded.reshape((frames.shape[0], frames.shape[1], Q_LEVELS * FRAME_SIZE)) output = MLP('FrameProcessor', FRAME_SIZE*Q_LEVELS, DIM, embedded) return output # frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) # frames *= lib.floatX(2) # output = MLP('FrameProcessor', FRAME_SIZE, DIM, frames) # return output
def Recurrence(processed_frames, h0, reset): """ processed_frames.shape: (batch size, n frames, DIM) h0.shape: (batch size, N_GRUS, DIM) reset.shape: () output.shape: (batch size, n frames, DIM) """ # print "warning no recurrence" # return T.zeros_like(processed_frames), h0 learned_h0 = lib.param( 'Recurrence.h0', numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM) learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) gru0 = lib.ops.LowMemGRU('Recurrence.GRU0', DIM, DIM, processed_frames, h0=h0[:, 0]) grus = [gru0] for i in xrange(1, N_GRUS): gru = lib.ops.LowMemGRU('Recurrence.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i]) grus.append(gru) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) return (grus[-1], last_hidden)
def softmax_and_sample(logits): old_shape = logits.shape flattened_logits = logits.reshape((-1, logits.shape[logits.ndim-1])) samples = T.cast( srng.multinomial(pvals=T.nnet.softmax(flattened_logits)), theano.config.floatX ).reshape(old_shape) return T.argmax(samples, axis=samples.ndim-1)
def Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None): if not isinstance(inputs, list): inputs = [inputs] if not isinstance(hidden_dims, list): hidden_dims = [hidden_dims] if h0s is None: h0s = [None]*len(hidden_dims) for i in xrange(len(hidden_dims)): if h0s[i] is None: h0_unbatched = lib.param( name + '.h0_' + str(i), numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX) ) num_batches = inputs[0].shape[1] h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i]) h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim) outputs, _ = theano.scan( step_fn, sequences=inputs, outputs_info=h0s, non_sequences=non_sequences ) return outputs
def gaussian_nll(x, mu, log_sigma): sigma_squared = T.exp(2*log_sigma) return ( lib.floatX(0.5*numpy.log(2*numpy.pi)) + (2*log_sigma) + ( ((x-mu)**2) / (2*sigma_squared) ) )
def softmax_and_sample(logits, temperature=1.): """ :temperature: default 1. For high temperatures (temperature -> +Inf), all actions have nearly the same probability and the lower the temperature, the more expected rewards affect the probability. For a low temperature (temperature -> 0+), the probability of the action with the highest expected reward (max operation) tends to 1. """ temperature = lib.floatX(temperature) ZEROX = lib.floatX(0.) assert temperature >= ZEROX, "`temperature` should be a non-negative value!" old_shape = logits.shape flattened_logits = logits.reshape((-1, logits.shape[logits.ndim-1])) if temperature == ZEROX: # Get max instead of (biased) sample. # Equivalent to directly get the argmax but with this it's easier to # extract the probabilities later on too. samples = T.nnet.softmax(flattened_logits) else: # > 0 flattened_logits /= temperature samples = T.cast( srng.multinomial(pvals=T.nnet.softmax(flattened_logits)), theano.config.floatX ) samples = samples.reshape(old_shape) return T.argmax(samples, axis=samples.ndim-1)
def __Recurrent(name, hidden_dims, step_fn, inputs, non_sequences=[], h0s=None): if not isinstance(inputs, list): inputs = [inputs] if not isinstance(hidden_dims, list): hidden_dims = [hidden_dims] if h0s is None: h0s = [None]*len(hidden_dims) for i in xrange(len(hidden_dims)): if h0s[i] is None: h0_unbatched = lib.param( name + '.h0_' + str(i), numpy.zeros((hidden_dims[i],), dtype=theano.config.floatX) ) num_batches = inputs[0].shape[1] h0s[i] = T.alloc(h0_unbatched, num_batches, hidden_dims[i]) h0s[i] = T.patternbroadcast(h0s[i], [False] * h0s[i].ndim) outputs, _ = theano.scan( step_fn, sequences=inputs, outputs_info=h0s, non_sequences=non_sequences ) return outputs
def T_one_hot(inp_tensor, n_classes): """ :todo: - Implement other methods from here: - Compare them for speed-wise for different sizes - Implement N_one_hot for Numpy version, with speed tests. Theano one-hot (1-of-k) from an input tensor of indecies. If the indecies are of the shape (a0, a1, ..., an) the output shape would be (a0, a1, ..., a2, n_classes). :params: - inp_tensor: any theano tensor with dtype int* as indecies and all of them between [0, n_classes-1]. - n_classes: number of classes which determines the output size. :usage: >>> idx = T.itensor3() >>> idx_val = numpy.array([[[0,1,2,3],[4,5,6,7]]], dtype='int32') >>> one_hot = T_one_hot(t, 8) >>> one_hot.eval({idx:idx_val}) >>> print out array([[[[ 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0.], [ 0., 0., 0., 1., 0., 0., 0., 0.]], [[ 0., 0., 0., 0., 1., 0., 0., 0.], [ 0., 0., 0., 0., 0., 1., 0., 0.], [ 0., 0., 0., 0., 0., 0., 1., 0.], [ 0., 0., 0., 0., 0., 0., 0., 1.]]]]) >>> print idx_val.shape, out.shape (1, 2, 4) (1, 2, 4, 8) """ flattened = inp_tensor.flatten() z = T.zeros((flattened.shape[0], n_classes), dtype=theano.config.floatX) one_hot = T.set_subtensor(z[T.arange(flattened.shape[0]), flattened], 1) out_shape = [inp_tensor.shape[i] for i in xrange(inp_tensor.ndim)] + [n_classes] one_hot = one_hot.reshape(out_shape) return one_hot
def relu(x): # Using T.nnet.relu gives me NaNs. No idea why. return T.switch(x > lib.floatX(0), x, lib.floatX(0))
def sample_from_softmax(softmax_var): #softmax_var assumed to be of shape (batch_size, num_classes) old_shape = softmax_var.shape softmax_var_reshaped = softmax_var.reshape((-1,softmax_var.shape[softmax_var.ndim-1])) return T.argmax( T.cast( srng.multinomial(pvals=softmax_var_reshaped), theano.config.floatX ).reshape(old_shape), axis = softmax_var.ndim-1 ) # inputs.shape: (batch size, length, input_dim)
def floatX(num): if theano.config.floatX == 'float32': return np.float32(num) else: raise Exception("{} type not supported".format(theano.config.floatX))
def binarize(images): """ Stochastically binarize values in [0, 1] by treating them as p-values of a Bernoulli distribution. """ return ( np.random.uniform(size=images.shape) < images ).astype(theano.config.floatX)
def init_weights(fan_in,fan_out,init='he'): def uniform(stdev, size): """uniform distribution with the given stdev and size""" return numpy.random.uniform( low=-stdev * numpy.sqrt(3), high=stdev * numpy.sqrt(3), size=size ).astype(theano.config.floatX) if init == 'lecun' or (init == None and fan_in != fan_out): weight_values = uniform(numpy.sqrt(1. / fan_in), (fan_in, fan_out)) elif init == 'he': weight_values = uniform(numpy.sqrt(2. / fan_in), (fan_in, fan_out)) elif init == 'orthogonal' or (init == None and fan_in == fan_out): # From lasagne def sample(shape): if len(shape) < 2: raise RuntimeError("Only shapes of length 2 or more are " "supported.") flat_shape = (shape[0], numpy.prod(shape[1:])) # TODO: why normal and not uniform? a = numpy.random.normal(0.0, 1.0, flat_shape) u, _, v = numpy.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) return q.astype(theano.config.floatX) weight_values = sample((fan_in, fan_out)) return weight_values
def Dense(name, input_dim, output_dim, inputs, bias=True, init=None, weightnorm=True,hidden_dim=None): weight_values = init_weights(input_dim,output_dim,init) weight = lib.param( name + '.W', weight_values ) batch_size = None if inputs.ndim==3: batch_size = inputs.shape[0] inputs = inputs.reshape((-1,input_dim)) if weightnorm: norm_values = numpy.linalg.norm(weight_values, axis=0) norms = lib.param( name + '.g', norm_values ) normed_weight = weight * (norms / weight.norm(2, axis=0)).dimshuffle('x', 0) result = T.dot(inputs, normed_weight) else: result = T.dot(inputs, weight) if bias: b = lib.param( name + '.b', numpy.zeros((output_dim,), dtype=theano.config.floatX) ) result += b result.name = name+".output" if batch_size!=None: return result.reshape((batch_size,hidden_dim,output_dim)) else: return result
def Embedding(name, n_symbols, output_dim, indices): vectors = lib.param( name, numpy.random.randn( n_symbols, output_dim ).astype(theano.config.floatX) ) output_shape = tuple(list(indices.shape) + [output_dim]) return vectors[indices.flatten()].reshape(output_shape)
def GRUStep(name, input_dim, hidden_dim, x_t, h_tm1): processed_input = lib.ops.Dense( name+'.Input', input_dim, 3 * hidden_dim, x_t ) gates = T.nnet.sigmoid( lib.ops.Dense( name+'.Recurrent_Gates', hidden_dim, 2 * hidden_dim, h_tm1, bias=False ) + processed_input[:, :2*hidden_dim] ) update = gates[:, :hidden_dim] reset = gates[:, hidden_dim:] scaled_hidden = reset * h_tm1 candidate = T.tanh( lib.ops.Dense( name+'.Recurrent_Candidate', hidden_dim, hidden_dim, scaled_hidden, bias=False, init='orthogonal' ) + processed_input[:, 2*hidden_dim:] ) one = lib.floatX(1.0) return (update * candidate) + ((one - update) * h_tm1)
def Conv1D(name, input_dim, output_dim, filter_size, inputs, apply_biases=True): """ inputs.shape: (batch size, height, input_dim) output.shape: (batch size, height, output_dim) * performs valid convs """ def uniform(stdev, size): """uniform distribution with the given stdev and size""" return numpy.random.uniform( low=-stdev * numpy.sqrt(3), high=stdev * numpy.sqrt(3), size=size ).astype(theano.config.floatX) filters = lib.param( name+'.Filters', uniform( 1./numpy.sqrt(input_dim * filter_size), # output dim, input dim, height, width (output_dim, input_dim, filter_size, 1) ) ) # conv2d takes inputs as (batch size, input channels, height[?], width[?]) inputs = inputs.reshape((inputs.shape[0], inputs.shape[1], 1, inputs.shape[2])) inputs = inputs.dimshuffle(0, 3, 1, 2) result = T.nnet.conv2d(inputs, filters, border_mode='valid', filter_flip=False) if apply_biases: biases = lib.param( name+'.Biases', numpy.zeros(output_dim, dtype=theano.config.floatX) ) result = result + biases[None, :, None, None] result = result.dimshuffle(0, 2, 3, 1) return result.reshape((result.shape[0], result.shape[1], result.shape[3]))
def Skew(inputs): """ input.shape: (batch size, HEIGHT, WIDTH, dim) """ buffer = T.zeros( (inputs.shape[0], inputs.shape[1], 2*inputs.shape[2] - 1, inputs.shape[3]), theano.config.floatX ) for i in xrange(HEIGHT): buffer = T.inc_subtensor(buffer[:, i, i:i+WIDTH, :], inputs[:,i,:,:]) return buffer
def big_frame_level_rnn(input_sequences, h0, reset): """ input_sequences.shape: (batch size, n big frames * BIG_FRAME_SIZE) h0.shape: (batch size, N_BIG_GRUS, BIG_DIM) reset.shape: () output[0].shape: (batch size, n frames, DIM) output[1].shape: same as h0.shape output[2].shape: (batch size, seq len, Q_LEVELS) """ learned_h0 = lib.param( 'BigFrameLevel.h0', numpy.zeros((N_BIG_GRUS, BIG_DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_BIG_GRUS, BIG_DIM) learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) frames = input_sequences.reshape(( input_sequences.shape[0], input_sequences.shape[1] / BIG_FRAME_SIZE, BIG_FRAME_SIZE )) # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2] # (a reasonable range to pass as inputs to the RNN) frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) frames *= lib.floatX(2) gru0 = lib.ops.LowMemGRU('BigFrameLevel.GRU0', BIG_FRAME_SIZE, BIG_DIM, frames, h0=h0[:, 0]) grus = [gru0] for i in xrange(1, N_BIG_GRUS): gru = lib.ops.LowMemGRU('BigFrameLevel.GRU'+str(i), BIG_DIM, BIG_DIM, grus[-1], h0=h0[:, i]) grus.append(gru) output = lib.ops.Linear( 'BigFrameLevel.Output', BIG_DIM, DIM * BIG_FRAME_SIZE / FRAME_SIZE, grus[-1] ) output = output.reshape((output.shape[0], output.shape[1] * BIG_FRAME_SIZE / FRAME_SIZE, DIM)) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) independent_preds = lib.ops.Linear( 'BigFrameLevel.IndependentPreds', BIG_DIM, Q_LEVELS * BIG_FRAME_SIZE, grus[-1] ) independent_preds = independent_preds.reshape((independent_preds.shape[0], independent_preds.shape[1] * BIG_FRAME_SIZE, Q_LEVELS)) return (output, last_hidden, independent_preds)
def frame_level_rnn(input_sequences, other_input, h0, reset): """ input_sequences.shape: (batch size, n frames * FRAME_SIZE) other_input.shape: (batch size, n frames, DIM) h0.shape: (batch size, N_GRUS, DIM) reset.shape: () output.shape: (batch size, n frames * FRAME_SIZE, DIM) """ learned_h0 = lib.param( 'FrameLevel.h0', numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM) learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) frames = input_sequences.reshape(( input_sequences.shape[0], input_sequences.shape[1] / FRAME_SIZE, FRAME_SIZE )) # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2] # (a reasonable range to pass as inputs to the RNN) frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) frames *= lib.floatX(2) gru_input = lib.ops.Linear('FrameLevel.InputExpand', FRAME_SIZE, DIM, frames) + other_input gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', DIM, DIM, gru_input, h0=h0[:, 0]) grus = [gru0] for i in xrange(1, N_GRUS): gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i]) grus.append(gru) output = lib.ops.Linear( 'FrameLevel.Output', DIM, FRAME_SIZE * DIM, grus[-1], initialization='he' ) output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM)) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) return (output, last_hidden)
def sample_level_rnn(input_sequences, h0, reset): """ input_sequences.shape: (batch size, seq len) h0.shape: (batch size, N_GRUS, DIM) reset.shape: () output.shape: (batch size, seq len, Q_LEVELS) """ learned_h0 = lib.param( 'SampleLevel.h0', numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) # Embedded inputs ################# FRAME_SIZE = Q_LEVELS frames = lib.ops.Embedding('SampleLevel.Embedding', Q_LEVELS, Q_LEVELS, input_sequences) # Real-valued inputs #################### # 'frames' of size 1 # FRAME_SIZE = 1 # frames = input_sequences.reshape(( # input_sequences.shape[0], # input_sequences.shape[1], # 1 # )) # # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2] # # (a reasonable range to pass as inputs to the RNN) # frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) # frames *= lib.floatX(2) gru0 = lib.ops.LowMemGRU('SampleLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0]) # gru0 = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU0FF', DIM, DIM, gru0, initialization='he')) grus = [gru0] for i in xrange(1, N_GRUS): gru = lib.ops.LowMemGRU('SampleLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i]) # gru = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU'+str(i)+'FF', DIM, DIM, gru, initialization='he')) grus.append(gru) # We apply the softmax later output = lib.ops.Linear( 'Output', N_GRUS*DIM, Q_LEVELS, T.concatenate(grus, axis=2) ) # output = lib.ops.Linear( # 'Output', # DIM, # Q_LEVELS, # grus[-1] # ) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) return (output, last_hidden)
def frame_level_rnn(input_sequences, h0, reset): """ input_sequences.shape: (batch size, n frames * FRAME_SIZE) h0.shape: (batch size, N_GRUS, DIM) reset.shape: () output.shape: (batch size, n frames * FRAME_SIZE, DIM) """ learned_h0 = lib.param( 'FrameLevel.h0', numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM) learned_h0 = T.patternbroadcast(learned_h0, [False] * learned_h0.ndim) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) frames = input_sequences.reshape(( input_sequences.shape[0], input_sequences.shape[1] / FRAME_SIZE, FRAME_SIZE )) # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2] # (a reasonable range to pass as inputs to the RNN) frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) frames *= lib.floatX(2) gru0 = lib.ops.LowMemGRU('FrameLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0]) grus = [gru0] for i in xrange(1, N_GRUS): gru = lib.ops.LowMemGRU('FrameLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i]) grus.append(gru) output = lib.ops.Linear( 'FrameLevel.Output', DIM, FRAME_SIZE * DIM, grus[-1], initialization='he' ) output = output.reshape((output.shape[0], output.shape[1] * FRAME_SIZE, DIM)) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) return (output, last_hidden)
def sample_level_rnn(input_sequences, h0, reset): """ input_sequences.shape: (batch size, seq len) h0.shape: (batch size, N_GRUS, DIM) reset.shape: () output.shape: (batch size, seq len, Q_LEVELS) """ learned_h0 = lib.param( 'SampleLevel.h0', numpy.zeros((N_GRUS, DIM), dtype=theano.config.floatX) ) learned_h0 = T.alloc(learned_h0, h0.shape[0], N_GRUS, DIM) h0 = theano.ifelse.ifelse(reset, learned_h0, h0) # Embedded inputs ################# # FRAME_SIZE = Q_LEVELS # frames = lib.ops.Embedding('SampleLevel.Embedding', Q_LEVELS, Q_LEVELS, input_sequences) # Real-valued inputs #################### # 'frames' of size 1 FRAME_SIZE = 1 frames = input_sequences.reshape(( input_sequences.shape[0], input_sequences.shape[1], 1 )) # # Rescale frames from ints in [0, Q_LEVELS) to floats in [-2, 2] # # (a reasonable range to pass as inputs to the RNN) # frames = (frames.astype('float32') / lib.floatX(Q_LEVELS/2)) - lib.floatX(1) # frames *= lib.floatX(2) gru0 = lib.ops.LowMemGRU('SampleLevel.GRU0', FRAME_SIZE, DIM, frames, h0=h0[:, 0]) # gru0 = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU0FF', DIM, DIM, gru0, initialization='he')) grus = [gru0] for i in xrange(1, N_GRUS): gru = lib.ops.LowMemGRU('SampleLevel.GRU'+str(i), DIM, DIM, grus[-1], h0=h0[:, i]) # gru = T.nnet.relu(lib.ops.Linear('SampleLevel.GRU'+str(i)+'FF', DIM, DIM, gru, initialization='he')) grus.append(gru) # We apply the softmax later output = lib.ops.Linear( 'Output', N_GRUS*DIM, 2, T.concatenate(grus, axis=2) ) # output = lib.ops.Linear( # 'Output', # DIM, # Q_LEVELS, # grus[-1] # ) last_hidden = T.stack([gru[:,-1] for gru in grus], axis=1) return (output, last_hidden)
def conv1d( name, input, input_dim, output_dim, filter_size, init = 'glorot', non_linearity = 'relu', bias = True ): import lasagne inp = input.dimshuffle(0,2,1,'x') if init == 'glorot': initializer = lasagne.init.GlorotUniform() elif init == 'he': initializer = lasagne.init.HeUniform() if non_linearity == 'gated': num_filters = 2*output_dim else: num_filters = output_dim W_shape = (num_filters, input_dim, filter_size, 1) if bias: bias_shape = (num_filters,) W = lib.param(name+".W", initializer.sample(W_shape)) if bias: b = lib.param(name+".b", lasagne.init.Constant(0.).sample(bias_shape)) conv_out = T.nnet.conv2d( inp, W, filter_flip= False, border_mode = 'valid' ) if bias: conv_out = conv_out + b[None,:,None, None] if non_linearity == 'gated': activation = gated_non_linerity elif non_linearity == 'relu': activation = T.nnet.relu elif non_linearity == 'elu': activation = lambda x : T.switch( x >= 0., x, T.exp(x) - floatX(1.)) elif non_linearity == 'identity': activation = lambda x: x else: raise NotImplementedError("{} non-linearity not implemented!".format(non_linearity)) output = conv_out output = output.reshape((output.shape[0], output.shape[1], output.shape[2])) output = output.dimshuffle(0,2,1) return output
def DilatedConv1D(name, input_dim, output_dim, filter_size, inputs, dilation, mask_type=None, apply_biases=True): """ inputs.shape: (batch size, length, input_dim) mask_type: None, 'a', 'b' output.shape: (batch size, length, output_dim) """ def uniform(stdev, size): """uniform distribution with the given stdev and size""" return numpy.random.uniform( low=-stdev * numpy.sqrt(3), high=stdev * numpy.sqrt(3), size=size ).astype(theano.config.floatX) filters_init = uniform( 1./numpy.sqrt(input_dim * filter_size), # output dim, input dim, height, width (output_dim, input_dim, filter_size, 1) ) if mask_type is not None: filters_init *= lib.floatX(numpy.sqrt(2.)) filters = lib.param( name+'.Filters', filters_init ) if mask_type is not None: mask = numpy.ones( (output_dim, input_dim, filter_size, 1), dtype=theano.config.floatX ) center = filter_size//2 for i in xrange(filter_size): if (i > center): mask[:, :, i, :] = 0. # if (mask_type=='a' and i == center): # mask[:, :, center] = 0. filters = filters * mask inputs = inputs.reshape((inputs.shape[0], inputs.shape[1], 1, inputs.shape[2])) # conv2d takes inputs as (batch size, input channels, height[?], width[?]) inputs = inputs.dimshuffle(0, 3, 1, 2) result = T.nnet.conv2d(inputs, filters, border_mode='half', filter_flip=False, filter_dilation=(dilation, 1)) if apply_biases: biases = lib.param( name+'.Biases', numpy.zeros(output_dim, dtype=theano.config.floatX) ) result = result + biases[None, :, None, None] result = result.dimshuffle(0, 2, 3, 1) return result.reshape((result.shape[0], result.shape[1], result.shape[3]))
def generate_and_save_samples(tag): lib.save_params(os.path.join(OUT_DIR, tag + "_params.pkl")) def save_images(images, filename, i = None): """images.shape: (batch, n channels, height, width)""" if i is not None: new_tag = "{}_{}".format(tag, i) else: new_tag = tag images = images.reshape((10,10,28,28)) images = images.transpose(1,2,0,3) images = images.reshape((10*28, 10*28)) image = scipy.misc.toimage(images, cmin=0.0, cmax=1.0) image.save('{}/{}_{}.jpg'.format(OUT_DIR, filename, new_tag)) latents = np.random.normal(size=(100, LATENT_DIM)) latents = latents.astype(theano.config.floatX) samples = np.zeros( (100, N_CHANNELS, HEIGHT, WIDTH), dtype=theano.config.floatX ) next_sample = samples.copy() t0 = time.time() for j in xrange(HEIGHT): for k in xrange(WIDTH): for i in xrange(N_CHANNELS): samples_p_value = sample_fn(latents, next_sample) next_sample[:, i, j, k] = binarize(samples_p_value)[:, i, j, k] samples[:, i, j, k] = samples_p_value[:, i, j, k] t1 = time.time() print("Time taken for generation {:.4f}".format(t1 - t0)) save_images(samples_p_value, 'samples')
def myGRU(name, input_dim, hidden_dim, inputs, h0=None): #inputs.shape = (batch_size,N_FRAMES,FRAME_SIZE) inputs = inputs.transpose(1,0,2) weight_values = lasagne.init.GlorotUniform().sample((input_dim+hidden_dim,2*hidden_dim)) W1 = lib.param( name+'.Gates.W', weight_values ) b1 = lib.param( name+'.Gates.b', np.ones(2*hidden_dim).astype(theano.config.floatX) ) weight_values = lasagne.init.GlorotUniform().sample((input_dim+hidden_dim,hidden_dim)) W2 = lib.param( name+'.Candidate.W', weight_values ) b2 = lib.param( name+'.Candidate.b', np.zeros(hidden_dim).astype(theano.config.floatX) ) def step(x_t, h_tm1): return recurrent_fn( x_t, h_tm1, name, input_dim, hidden_dim, W1,b1,W2,b2 ) outputs, _ = theano.scan( step, sequences=[inputs], outputs_info=[h0], ) out = outputs.dimshuffle(1,0,2) out.name = name+'.output' return out
def DiagonalLSTM(name, input_dim, inputs): """ inputs.shape: (batch size, height, width, input_dim) outputs.shape: (batch size, height, width, DIM) """ inputs = Skew(inputs) input_to_state = Conv2D(name+'.InputToState', input_dim, 4*DIM, 1, inputs, mask_type='b') batch_size = inputs.shape[0] c0_unbatched = lib.param( name + '.c0', numpy.zeros((HEIGHT, DIM), dtype=theano.config.floatX) ) c0 = T.alloc(c0_unbatched, batch_size, HEIGHT, DIM) h0_unbatched = lib.param( name + '.h0', numpy.zeros((HEIGHT, DIM), dtype=theano.config.floatX) ) h0 = T.alloc(h0_unbatched, batch_size, HEIGHT, DIM) def step_fn(current_input_to_state, prev_c, prev_h): # all args have shape (batch size, height, DIM) # TODO consider learning this padding prev_h = T.concatenate([ T.zeros((batch_size, 1, DIM), theano.config.floatX), prev_h ], axis=1) state_to_state = Conv1D(name+'.StateToState', DIM, 4*DIM, 2, prev_h, apply_biases=False) gates = current_input_to_state + state_to_state o_f_i = T.nnet.sigmoid(gates[:,:,:3*DIM]) o = o_f_i[:,:,0*DIM:1*DIM] f = o_f_i[:,:,1*DIM:2*DIM] i = o_f_i[:,:,2*DIM:3*DIM] g = T.tanh(gates[:,:,3*DIM:4*DIM]) new_c = (f * prev_c) + (i * g) new_h = o * T.tanh(new_c) return (new_c, new_h) outputs, _ = theano.scan( step_fn, sequences=input_to_state.dimshuffle(2,0,1,3), outputs_info=[c0, h0] ) all_cs = outputs[0].dimshuffle(1,2,0,3) all_hs = outputs[1].dimshuffle(1,2,0,3) return Unskew(all_hs)
