我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.check_numerics()。
def get_acceptance_rate(q, p, new_q, new_p, log_posterior, mass, data_axes): old_hamiltonian, old_log_prob = hamiltonian( q, p, log_posterior, mass, data_axes) new_hamiltonian, new_log_prob = hamiltonian( new_q, new_p, log_posterior, mass, data_axes) old_log_prob = tf.check_numerics( old_log_prob, 'HMC: old_log_prob has numeric errors! Try better initialization.') acceptance_rate = tf.exp( tf.minimum(-new_hamiltonian + old_hamiltonian, 0.0)) is_finite = tf.logical_and(tf.is_finite(acceptance_rate), tf.is_finite(new_log_prob)) acceptance_rate = tf.where(is_finite, acceptance_rate, tf.zeros_like(acceptance_rate)) return old_hamiltonian, new_hamiltonian, old_log_prob, new_log_prob, \ acceptance_rate
def __init__(self, rate, dtype=None, group_ndims=0, check_numerics=False, **kwargs): self._rate = tf.convert_to_tensor(rate) param_dtype = assert_same_float_dtype( [(self._rate, 'Poisson.rate')]) if dtype is None: dtype = tf.int32 assert_same_float_and_int_dtype([], dtype) self._check_numerics = check_numerics super(Poisson, self).__init__( dtype=dtype, param_dtype=param_dtype, is_continuous=False, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): logits = self.logits n = tf.cast(self.n_experiments, self.param_dtype) given = tf.cast(given, self.param_dtype) log_1_minus_p = -tf.nn.softplus(logits) lgamma_n_plus_1 = tf.lgamma(n + 1) lgamma_given_plus_1 = tf.lgamma(given + 1) lgamma_n_minus_given_plus_1 = tf.lgamma(n - given + 1) if self._check_numerics: lgamma_given_plus_1 = tf.check_numerics( lgamma_given_plus_1, "lgamma(given + 1)") lgamma_n_minus_given_plus_1 = tf.check_numerics( lgamma_n_minus_given_plus_1, "lgamma(n - given + 1)") return lgamma_n_plus_1 - lgamma_n_minus_given_plus_1 - \ lgamma_given_plus_1 + given * logits + n * log_1_minus_p
def _log_prob(self, given): temperature, logits = self.path_param(self.temperature), \ self.path_param(self.logits) log_given = tf.log(given) log_1_minus_given = tf.log(1 - given) log_temperature = tf.log(temperature) if self._check_numerics: log_given = tf.check_numerics(log_given, "log(given)") log_1_minus_given = tf.check_numerics( log_1_minus_given, "log(1 - given)") log_temperature = tf.check_numerics( log_temperature, "log(temperature)") logistic_given = log_given - log_1_minus_given temp = temperature * logistic_given - logits return log_temperature - log_given - log_1_minus_given + \ temp - 2 * tf.nn.softplus(temp)
def build_graph(self, weights, loss=None, optimizer=None, norm=False, batch_size=None, grad_ys=None): if loss is not None: gradients = tf.gradients(loss.node, list(utils.Utils.flatten(weights.node)), grad_ys) gradients = [tf.check_numerics(g, 'gradient_%d' % i) for i, g in enumerate(gradients)] if batch_size is not None: gradients = [g / float(batch_size) for g in gradients] # store gradients global norm before clipping self.global_norm = tf.global_norm(gradients) # clip gradients after global norm has been stored if norm: gradients, _ = tf.clip_by_global_norm(gradients, norm) self.calculate = graph.TfNode(utils.Utils.reconstruct(gradients, weights.node)) if optimizer is not None: self.ph_gradients = graph.Placeholders(weights) self.apply = graph.TfNode(optimizer.node.apply_gradients( utils.Utils.izip(self.ph_gradients.checked, weights.node)))
def build_graph(self, dtype, shape=None, name=None): """Assemble one placeholder. Args: shape: The shape of the tensor to be fed (optional). If the shape is not specified, you can feed a tensor of any shape. dtype: The type of elements in the placeholder to be fed. name: A name for the placeholder (optional). Returns: placeholder of given shape and data type """ ph = tf.placeholder(self.DTYPE[dtype], shape=shape, name=name) if dtype not in [np.int32, np.int64]: self.checked = tf.check_numerics(ph, '') return ph
def __init__(self, length_scale=1.0, magnitude=1.0, check_numerics=True, debug=False): assert np.isscalar(length_scale) assert np.isscalar(magnitude) assert length_scale > 0 and magnitude > 0 self.length_scale = length_scale self.magnitude = magnitude self.check_numerics = check_numerics self.debug = debug self.X_train = None self.y_train = None self.xy_ = None self.K = None self.graph = None self.vars = None self.ops = None
def fully_connected(in_tensor, layer_name, out_chan, trainable=True): with tf.variable_scope(layer_name): in_size = in_tensor.get_shape().as_list() assert len(in_size) == 2, 'Input to a fully connected layer must be a vector.' weights_shape = [in_size[1], out_chan] # weight matrix weights = tf.get_variable('weights', weights_shape, tf.float32, tf.contrib.layers.xavier_initializer(), trainable=trainable) weights = tf.check_numerics(weights, 'weights: %s' % layer_name) # bias biases = tf.get_variable('biases', [out_chan], tf.float32, tf.constant_initializer(0.0001), trainable=trainable) biases = tf.check_numerics(biases, 'biases: %s' % layer_name) out_tensor = tf.matmul(in_tensor, weights) + biases return out_tensor
def v_from_u(u, log_alpha, force_same=True, b=None, v_prime=None): u_prime = tf.nn.sigmoid(-log_alpha) if not force_same: v = b*(u_prime+v_prime*(1-u_prime)) + (1-b)*v_prime*u_prime else: v_1 = (u - u_prime) / safe_clip(1 - u_prime) v_1 = tf.clip_by_value(v_1, 0, 1) v_1 = tf.stop_gradient(v_1) v_1 = v_1 * (1 - u_prime) + u_prime v_0 = u / safe_clip(u_prime) v_0 = tf.clip_by_value(v_0, 0, 1) v_0 = tf.stop_gradient(v_0) v_0 = v_0 * u_prime v = tf.where(u > u_prime, v_1, v_0) v = tf.check_numerics(v, 'v sampling is not numerically stable.') if force_same: v = v + tf.stop_gradient(-v + u) # v and u are the same up to numerical errors return v
def v_from_u(u, log_alpha, force_same=True): # Lovingly copied from https://github.com/tensorflow/models/blob/master/research/rebar/rebar.py u_prime = tf.nn.sigmoid(-log_alpha) v_1 = (u - u_prime) / safe_clip(1 - u_prime) v_1 = tf.clip_by_value(v_1, 0, 1) v_1 = tf.stop_gradient(v_1) v_1 = v_1 * (1 - u_prime) + u_prime v_0 = u / safe_clip(u_prime) v_0 = tf.clip_by_value(v_0, 0, 1) v_0 = tf.stop_gradient(v_0) v_0 = v_0 * u_prime v = tf.where(u > u_prime, v_1, v_0) v = tf.check_numerics(v, 'v sampling is not numerically stable.') if force_same: v = v + tf.stop_gradient(-v + u) # v and u are the same up to numerical errors return v
def _u_to_v(self, log_alpha, u, eps = 1e-8): """Convert u to tied randomness in v.""" u_prime = tf.nn.sigmoid(-log_alpha) # g(u') = 0 v_1 = (u - u_prime) / tf.clip_by_value(1 - u_prime, eps, 1) v_1 = tf.clip_by_value(v_1, 0, 1) v_1 = tf.stop_gradient(v_1) v_1 = v_1*(1 - u_prime) + u_prime v_0 = u / tf.clip_by_value(u_prime, eps, 1) v_0 = tf.clip_by_value(v_0, 0, 1) v_0 = tf.stop_gradient(v_0) v_0 = v_0 * u_prime v = tf.where(u > u_prime, v_1, v_0) v = tf.check_numerics(v, 'v sampling is not numerically stable.') v = v + tf.stop_gradient(-v + u) # v and u are the same up to numerical errors return v
def qa_rnn(config, is_training, input_seqs, input_masks, xvector=None): """Model that takes as input several input sequences, output the encoded vector for each of the sequence. Args: is_training: boolean. Indictates if the model is used for training or not. input_seq: 2-D list of tensors, each is a [batch_size * embed_size] tensor. """ embed_seqs, atten_seqs, all_output_seqs = \ sentences_encoding(config, is_training, input_seqs, input_masks, xvector) if NUMERIC_CHECK: embed_seqs = tf.check_numerics(embed_seqs, 'qa_rnn output embedding numeric error') atten_seqs = tf.check_numerics(atten_seqs, 'qa_rnn output attention numeric error') all_output_seqs = tf.check_numerics(all_output_seqs, 'qa_rnn output numeric error') return embed_seqs, atten_seqs, all_output_seqs
def _log_prob(self, given): mean, logstd = self.path_param(self.mean),\ self.path_param(self.logstd) c = -0.5 * np.log(2 * np.pi) precision = tf.exp(-2 * logstd) if self._check_numerics: precision = tf.check_numerics(precision, "precision") return c - logstd - 0.5 * precision * tf.square(given - mean)
def _log_prob(self, given): mean, logstd = self.path_param(self.mean), \ self.path_param(self.logstd) c = -0.5 * (np.log(2.0) + np.log(np.pi)) precision = tf.exp(-2.0 * logstd) if self._check_numerics: precision = tf.check_numerics(precision, "precision") mask = tf.log(tf.cast(given >= 0., dtype=precision.dtype)) return (c - (logstd + 0.5 * precision * tf.square(given - mean)) + \ tf.nn.softplus(-2.0 * mean * given * precision)) + mask
def _log_prob(self, given): log_p = tf.log(self._prob(given)) if self._check_numerics: log_p = tf.check_numerics(log_p, "log_p") return log_p
def _prob(self, given): mask = tf.cast(tf.logical_and(tf.less_equal(self.minval, given), tf.less(given, self.maxval)), self.dtype) p = 1. / (self.maxval - self.minval) if self._check_numerics: p = tf.check_numerics(p, "p") return p * mask
def __init__(self, alpha, beta, group_ndims=0, check_numerics=False, **kwargs): self._alpha = tf.convert_to_tensor(alpha) self._beta = tf.convert_to_tensor(beta) dtype = assert_same_float_dtype( [(self._alpha, 'Gamma.alpha'), (self._beta, 'Gamma.beta')]) try: tf.broadcast_static_shape(self._alpha.get_shape(), self._beta.get_shape()) except ValueError: raise ValueError( "alpha and beta should be broadcastable to match each " "other. ({} vs. {})".format( self._alpha.get_shape(), self._beta.get_shape())) self._check_numerics = check_numerics super(Gamma, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): alpha, beta = self.alpha, self.beta log_given = tf.log(given) log_beta = tf.log(beta) lgamma_alpha = tf.lgamma(alpha) if self._check_numerics: log_given = tf.check_numerics(log_given, "log(given)") log_beta = tf.check_numerics(log_beta, "log(beta)") lgamma_alpha = tf.check_numerics(lgamma_alpha, "lgamma(alpha)") return alpha * log_beta - lgamma_alpha + (alpha - 1) * log_given - \ beta * given
def __init__(self, alpha, beta, dtype=None, group_ndims=0, check_numerics=False, **kwargs): self._alpha = tf.convert_to_tensor(alpha) self._beta = tf.convert_to_tensor(beta) dtype = assert_same_float_dtype( [(self._alpha, 'Beta.alpha'), (self._beta, 'Beta.beta')]) try: tf.broadcast_static_shape(self._alpha.get_shape(), self._beta.get_shape()) except ValueError: raise ValueError( "alpha and beta should be broadcastable to match each " "other. ({} vs. {})".format( self._alpha.get_shape(), self._beta.get_shape())) self._check_numerics = check_numerics super(Beta, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): rate = self.rate given = tf.cast(given, self.param_dtype) log_rate = tf.log(rate) lgamma_given_plus_1 = tf.lgamma(given + 1) if self._check_numerics: log_rate = tf.check_numerics(log_rate, "log(rate)") lgamma_given_plus_1 = tf.check_numerics( lgamma_given_plus_1, "lgamma(given + 1)") return given * log_rate - rate - lgamma_given_plus_1
def __init__(self, alpha, beta, group_ndims=0, check_numerics=False, **kwargs): self._alpha = tf.convert_to_tensor(alpha) self._beta = tf.convert_to_tensor(beta) dtype = assert_same_float_dtype( [(self._alpha, 'InverseGamma.alpha'), (self._beta, 'InverseGamma.beta')]) try: tf.broadcast_static_shape(self._alpha.get_shape(), self._beta.get_shape()) except ValueError: raise ValueError( "alpha and beta should be broadcastable to match each " "other. ({} vs. {})".format( self._alpha.get_shape(), self._beta.get_shape())) self._check_numerics = check_numerics super(InverseGamma, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def __init__(self, loc, scale, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._loc = tf.convert_to_tensor(loc) self._scale = tf.convert_to_tensor(scale) dtype = assert_same_float_dtype( [(self._loc, 'Laplace.loc'), (self._scale, 'Laplace.scale')]) try: tf.broadcast_static_shape(self._loc.get_shape(), self._scale.get_shape()) except ValueError: raise ValueError( "loc and scale should be broadcastable to match each " "other. ({} vs. {})".format( self._loc.get_shape(), self._scale.get_shape())) self._check_numerics = check_numerics super(Laplace, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): loc, scale = self.path_param(self.loc),\ self.path_param(self.scale) log_scale = tf.log(scale) if self._check_numerics: log_scale = tf.check_numerics(log_scale, "log(scale)") return -np.log(2.) - log_scale - tf.abs(given - loc) / scale
def __init__(self, temperature, logits, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._logits = tf.convert_to_tensor(logits) self._temperature = tf.convert_to_tensor(temperature) param_dtype = assert_same_float_dtype( [(self._logits, 'BinConcrete.logits'), (self._temperature, 'BinConcrete.temperature')]) self._temperature = assert_scalar( self._temperature, 'BinConcrete.temperature') self._check_numerics = check_numerics super(BinConcrete, self).__init__( dtype=param_dtype, param_dtype=param_dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def __init__(self, alpha, group_ndims=0, check_numerics=False, **kwargs): self._alpha = tf.convert_to_tensor(alpha) dtype = assert_same_float_dtype( [(self._alpha, 'Dirichlet.alpha')]) static_alpha_shape = self._alpha.get_shape() shape_err_msg = "alpha should have rank >= 1." cat_err_msg = "n_categories (length of the last axis " \ "of alpha) should be at least 2." if static_alpha_shape and (static_alpha_shape.ndims < 1): raise ValueError(shape_err_msg) elif static_alpha_shape and ( static_alpha_shape[-1].value is not None): self._n_categories = static_alpha_shape[-1].value if self._n_categories < 2: raise ValueError(cat_err_msg) else: _assert_shape_op = tf.assert_rank_at_least( self._alpha, 1, message=shape_err_msg) with tf.control_dependencies([_assert_shape_op]): self._alpha = tf.identity(self._alpha) self._n_categories = tf.shape(self._alpha)[-1] _assert_cat_op = tf.assert_greater_equal( self._n_categories, 2, message=cat_err_msg) with tf.control_dependencies([_assert_cat_op]): self._alpha = tf.identity(self._alpha) self._check_numerics = check_numerics super(Dirichlet, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): given, alpha = maybe_explicit_broadcast( given, self.alpha, 'given', 'alpha') lbeta_alpha = tf.lbeta(alpha) # fix of no static shape inference for tf.lbeta if alpha.get_shape(): lbeta_alpha.set_shape(alpha.get_shape()[:-1]) log_given = tf.log(given) if self._check_numerics: lbeta_alpha = tf.check_numerics(lbeta_alpha, "lbeta(alpha)") log_given = tf.check_numerics(log_given, "log(given)") log_p = -lbeta_alpha + tf.reduce_sum((alpha - 1) * log_given, -1) return log_p
def __init__(self, temperature, logits, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._logits = tf.convert_to_tensor(logits) self._temperature = tf.convert_to_tensor(temperature) param_dtype = assert_same_float_dtype( [(self._logits, 'ExpConcrete.logits'), (self._temperature, 'ExpConcrete.temperature')]) self._logits = assert_rank_at_least_one( self._logits, 'ExpConcrete.logits') self._n_categories = get_shape_at(self._logits, -1) self._temperature = assert_scalar( self._temperature, 'ExpConcrete.temperature') self._check_numerics = check_numerics super(ExpConcrete, self).__init__( dtype=param_dtype, param_dtype=param_dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def _log_prob(self, given): logits, temperature = self.path_param(self.logits),\ self.path_param(self.temperature) n = tf.cast(self.n_categories, self.dtype) log_temperature = tf.log(temperature) if self._check_numerics: log_temperature = tf.check_numerics( log_temperature, "log(temperature)") temp = logits - temperature * given return tf.lgamma(n) + (n - 1) * log_temperature + \ tf.reduce_sum(temp, axis=-1) - \ n * tf.reduce_logsumexp(temp, axis=-1)
def __init__(self, temperature, logits, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._logits = tf.convert_to_tensor(logits) self._temperature = tf.convert_to_tensor(temperature) param_dtype = assert_same_float_dtype( [(self._logits, 'Concrete.logits'), (self._temperature, 'Concrete.temperature')]) self._logits = assert_rank_at_least_one( self._logits, 'Concrete.logits') self._n_categories = get_shape_at(self._logits, -1) self._temperature = assert_scalar( self._temperature, 'Concrete.temperature') self._check_numerics = check_numerics super(Concrete, self).__init__( dtype=param_dtype, param_dtype=param_dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def clip_gradients_by_norm(grads_and_vars, add_to_summary=True): if add_to_summary: for grad, var in grads_and_vars: if grad is not None: variable_summaries(grad, 'grad/{}'.format(var.name[:-2])) # Clip by norm. Grad can be null when not training some modules. with tf.name_scope('clip_gradients_by_norm'): grads_and_vars = [ ( tf.check_numerics( tf.clip_by_norm(gv[0], 10.), 'Invalid gradient' ), gv[1] ) if gv[0] is not None else gv for gv in grads_and_vars ] if add_to_summary: for grad, var in grads_and_vars: if grad is not None: variable_summaries( grad, 'clipped_grad/{}'.format(var.name[:-2])) return grads_and_vars
def model(self, input_vectors, input_gene, input_variation, output_label, batch_size, embedding_size=EMBEDDINGS_SIZE, output_classes=9, learning_rate_initial=D2V_DOC_LEARNING_RATE_INITIAL, learning_rate_decay=D2V_DOC_LEARNING_RATE_DECAY, learning_rate_decay_steps=D2V_DOC_LEARNING_RATE_DECAY_STEPS): self.global_step = training_util.get_or_create_global_step() logits, targets = doc2vec_prediction_model(input_vectors, input_gene, input_variation, output_label, batch_size, is_training=True, embedding_size=embedding_size, output_classes=output_classes) self.prediction = tf.nn.softmax(logits) self.loss = tf.nn.softmax_cross_entropy_with_logits(labels=targets, logits=logits) self.loss = tf.reduce_mean(self.loss) tf.summary.scalar('loss', self.loss) # learning rate & optimizer self.learning_rate = tf.train.exponential_decay(learning_rate_initial, self.global_step, learning_rate_decay_steps, learning_rate_decay, staircase=True, name='learning_rate') tf.summary.scalar('learning_rate', self.learning_rate) sgd = tf.train.GradientDescentOptimizer(self.learning_rate) self.optimizer = sgd.minimize(self.loss, global_step=self.global_step) # metrics self.metrics = metrics.single_label(self.prediction, targets) # saver to save the model self.saver = tf.train.Saver() # check a nan value in the loss self.loss = tf.check_numerics(self.loss, 'loss is nan') return None
def model(self, input_text_begin, input_text_end, gene, variation, expected_labels, batch_size, vocabulary_size=VOCABULARY_SIZE, embeddings_size=EMBEDDINGS_SIZE, output_classes=9): # embeddings embeddings = _load_embeddings(vocabulary_size, embeddings_size) # global step self.global_step = training_util.get_or_create_global_step() # model with slim.arg_scope(self.text_classification_model.model_arg_scope()): outputs = self.text_classification_model.model(input_text_begin, input_text_end, gene, variation, output_classes, embeddings=embeddings, batch_size=batch_size) # loss targets = self.text_classification_model.targets(expected_labels, output_classes) self.loss = self.text_classification_model.loss(targets, outputs) tf.summary.scalar('loss', self.loss) # learning rate self.optimizer, self.learning_rate = \ self.text_classification_model.optimize(self.loss, self.global_step) if self.learning_rate is not None: tf.summary.scalar('learning_rate', self.learning_rate) # metrics self.metrics = metrics.single_label(outputs['prediction'], targets) # saver to save the model self.saver = tf.train.Saver() # check a nan value in the loss self.loss = tf.check_numerics(self.loss, 'loss is nan') return None
def build_graph(self, *layers): weights = [layer.weight.node for layer in layers] self.ph_weights = graph.Placeholders(variables=graph.TfNode(weights)) self.assign = graph.TfNode([tf.assign(variable, value) for variable, value in utils.Utils.izip(weights, self.ph_weights.checked)]) self.check = graph.TfNode(tf.group(*[tf.check_numerics(w, 'weight_%d' % i) for i, w in enumerate(utils.Utils.flatten(weights))])) self.global_norm = tf.global_norm(list(utils.Utils.flatten(weights))) return weights
def build_graph(self, variables): phs = utils.Utils.map(variables.node, lambda v: tf.placeholder(shape=v.get_shape(), dtype=v.dtype)) self.checked = utils.Utils.map(phs, lambda ph: tf.check_numerics(ph, '')) return phs
def train(self, batch): _, _, l = tf.get_default_session().run([self.check_numerics, self.train_op, self.loss], feed_dict={self.input_state: batch.state_1, self.input_action: batch.action, self.reward: batch.reward, self.terminal_mask: batch.terminal_mask, self.input_state_2: batch.state_2, base_network.IS_TRAINING: True}) return l
def _calc_scores(self, attention_scores): """ Combine exact match scores & attention scores :param attention_scores: [batch_size, max_question_length, table_size] :return: """ with tf.variable_scope("scoring"): scores = attention_scores tf.check_numerics(scores, message="Score Nan...") softmax_scores = tf.nn.softmax(scores, dim=-1) tf.check_numerics(softmax_scores, message="Softmax Score Nan...") return softmax_scores
def train(self, batch): flip_horizontally = np.random.random() < 0.5 if VERBOSE_DEBUG: print "batch.action" print batch.action.T print "batch.reward", batch.reward.T print "batch.terminal_mask", batch.terminal_mask.T print "flip_horizontally", flip_horizontally print "weights", batch.weight.T values = tf.get_default_session().run([self._l_values, self.value_net.value, self.advantage, self.target_value_net.value, self.print_gradient_norms], feed_dict={self.input_state: batch.state_1, self.input_action: batch.action, self.reward: batch.reward, self.terminal_mask: batch.terminal_mask, self.input_state_2: batch.state_2, self.importance_weight: batch.weight, base_network.IS_TRAINING: True, base_network.FLIP_HORIZONTALLY: flip_horizontally}) values = [np.squeeze(v) for v in values] print "_l_values", values[0].T print "value_net.value ", values[1].T print "advantage ", values[2].T print "target_value_net.value ", values[3].T _, _, l = tf.get_default_session().run([self.check_numerics, self.train_op, self.loss], feed_dict={self.input_state: batch.state_1, self.input_action: batch.action, self.reward: batch.reward, self.terminal_mask: batch.terminal_mask, self.input_state_2: batch.state_2, self.importance_weight: batch.weight, base_network.IS_TRAINING: True, base_network.FLIP_HORIZONTALLY: flip_horizontally}) return l
def detect(sess, model, names, image, path): preprocess = eval(args.preprocess) _, height, width, _ = image.get_shape().as_list() _image = read_image(path) image_original = np.array(np.uint8(_image)) if len(image_original.shape) == 2: image_original = np.repeat(np.expand_dims(image_original, -1), 3, 2) image_height, image_width, _ = image_original.shape image_std = preprocess(np.array(np.uint8(_image.resize((width, height)))).astype(np.float32)) feed_dict = {image: np.expand_dims(image_std, 0)} tensors = [model.conf, model.xy_min, model.xy_max] conf, xy_min, xy_max = sess.run([tf.check_numerics(t, t.op.name) for t in tensors], feed_dict=feed_dict) boxes = utils.postprocess.non_max_suppress(conf[0], xy_min[0], xy_max[0], args.threshold, args.threshold_iou) scale = [image_width / model.cell_width, image_height / model.cell_height] fig = plt.figure() ax = fig.gca() ax.imshow(image_original) colors = [prop['color'] for _, prop in zip(names, itertools.cycle(plt.rcParams['axes.prop_cycle']))] cnt = 0 for _conf, _xy_min, _xy_max in boxes: index = np.argmax(_conf) if _conf[index] > args.threshold: wh = _xy_max - _xy_min _xy_min = _xy_min * scale _wh = wh * scale linewidth = min(_conf[index] * 10, 3) ax.add_patch(patches.Rectangle(_xy_min, _wh[0], _wh[1], linewidth=linewidth, edgecolor=colors[index], facecolor='none')) ax.annotate(names[index] + ' (%.1f%%)' % (_conf[index] * 100), _xy_min, color=colors[index]) cnt += 1 fig.canvas.set_window_title('%d objects detected' % cnt) ax.set_xticks([]) ax.set_yticks([]) return fig
def main(): model = config.get('config', 'model') yolo = importlib.import_module('model.' + model) width = config.getint(model, 'width') height = config.getint(model, 'height') preprocess = getattr(importlib.import_module('detect'), args.preprocess) with tf.Session() as sess: ph_image = tf.placeholder(tf.float32, [1, height, width, 3], name='ph_image') builder = yolo.Builder(args, config) builder(ph_image) global_step = tf.contrib.framework.get_or_create_global_step() model_path = tf.train.latest_checkpoint(utils.get_logdir(config)) tf.logging.info('load ' + model_path) slim.assign_from_checkpoint_fn(model_path, tf.global_variables())(sess) tf.logging.info('global_step=%d' % sess.run(global_step)) tensors = [builder.model.conf, builder.model.xy_min, builder.model.xy_max] tensors = [tf.check_numerics(t, t.op.name) for t in tensors] cap = cv2.VideoCapture(0) try: while True: ret, image_bgr = cap.read() assert ret image_height, image_width, _ = image_bgr.shape scale = [image_width / builder.model.cell_width, image_height / builder.model.cell_height] image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB) image_std = np.expand_dims(preprocess(cv2.resize(image_rgb, (width, height))).astype(np.float32), 0) feed_dict = {ph_image: image_std} conf, xy_min, xy_max = sess.run(tensors, feed_dict) boxes = utils.postprocess.non_max_suppress(conf[0], xy_min[0], xy_max[0], args.threshold, args.threshold_iou) for _conf, _xy_min, _xy_max in boxes: index = np.argmax(_conf) if _conf[index] > args.threshold: _xy_min = (_xy_min * scale).astype(np.int) _xy_max = (_xy_max * scale).astype(np.int) cv2.rectangle(image_bgr, tuple(_xy_min), tuple(_xy_max), (255, 0, 255), 3) cv2.putText(image_bgr, builder.names[index] + ' (%.1f%%)' % (_conf[index] * 100), tuple(_xy_min), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) cv2.imshow('detection', image_bgr) cv2.waitKey(1) finally: cv2.destroyAllWindows() cap.release()
def cross_entropy_indirect(logprops, name): loss = tf.reduce_mean(-logprops, axis=0) loss = tf.check_numerics(loss, f'check/cross_entropy/{name}') return loss
def cross_entropy_direct(logits, target, name): loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=target) loss *= tf.cast(tf.not_equal(target, tf.zeros_like(target)), tf.float32) batch_loss = tf.reduce_sum(loss, axis=1) batch_loss = tf.reduce_mean(batch_loss, axis=0) batch_loss = tf.check_numerics(batch_loss, f'check/cross_entropy/{name}') return batch_loss
def _mapper(self, grad, var): # this is very slow... #op = tf.Assert(tf.reduce_all(tf.is_finite(var)), [var], summarize=100) grad = tf.check_numerics(grad, 'CheckGradient') return grad
def forward_and_jacobian(self, x, sum_log_det_jacobians, z): with tf.variable_scope(self.name): xs = int_shape(x) b = self.get_mask(xs, self.mask_type) # masked half of x x1 = x * b l,m = self.function_l_m(x1, b) y = x1 + tf.mul(-b+1.0, x*tf.check_numerics(tf.exp(l), "exp has NaN") + m) log_det_jacobian = tf.reduce_sum(l, [1,2,3]) sum_log_det_jacobians += log_det_jacobian return y,sum_log_det_jacobians, z
def _create_reparam_variables(self, eps=1e-8): # noise for generating z u = tf.random_uniform([self.n_samples, self.dim], dtype=tf.float32) log_alpha = self._log_alpha # logistic reparameterization z = g(u, log_alpha) z = log_alpha + safe_log_prob(u) - safe_log_prob(1 - u) # b = H(z) b = tf.to_float(tf.stop_gradient(z > 0)) # g(u', log_alpha) = 0 u_prime = tf.nn.sigmoid(-log_alpha) v_1 = (u - u_prime) / tf.clip_by_value(1 - u_prime, eps, 1) v_1 = tf.clip_by_value(v_1, 0, 1) v_1 = tf.stop_gradient(v_1) v_1 = v_1 * (1 - u_prime) + u_prime v_0 = u / tf.clip_by_value(u_prime, eps, 1) v_0 = tf.clip_by_value(v_0, 0, 1) v_0 = tf.stop_gradient(v_0) v_0 = v_0 * u_prime v = tf.where(u > u_prime, v_1, v_0) v = tf.check_numerics(v, 'v sampling is not numerically stable.') v = v + tf.stop_gradient(-v + u) # v and u are the same up to numerical errors tf.summary.histogram("u-v", u-v) z_tilde = log_alpha + safe_log_prob(v) - safe_log_prob(1 - v) self.b = b self.z = z self.z_tilde = z_tilde
def embed_tokens(self, is_training, config, embedding_initializer): """Embedds input tokens. """ vocab_size = config.vocab_size size = config.word_embed_size max_question_length = self.max_question_length max_sentence_length = self.max_sentence_length max_sentence_num = self.max_sentence_num with tf.variable_scope("embed"): with tf.device("/cpu:0"): embedding = tf.get_variable( "embedding_mat", [vocab_size, size], initializer=embedding_initializer, dtype=config.data_type, trainable=False # Continue to train pretrained word2vec # trainable=True # Continue to train pretrained word2vec ) self._embedding = embedding embed_question= [] for i in xrange(max_question_length): embed_question.append( tf.nn.embedding_lookup(embedding, self._input_question[i])) if is_training and config.w_embed_keep_prob < 1: embed_question[i] = tf.nn.dropout(embed_question[i], config.w_embed_keep_prob) if NUMERIC_CHECK: embed_question[i] = \ tf.check_numerics(embed_question[i], "embed_question[{}][{}] numeric error".format(i)) embed_sentences = [] for i in xrange(max_sentence_num): embed_sentences.append([]) for j in xrange(max_sentence_length): embed_sentences[i].append( tf.nn.embedding_lookup(embedding, self._input_sentences[i][j])) if is_training and config.w_embed_keep_prob < 1: embed_sentences[i][j] = tf.nn.dropout(embed_sentences[i][j], config.w_embed_keep_prob) if NUMERIC_CHECK: embed_sentences[i][j] = \ tf.check_numerics(embed_sentences[i][j], "embed_sentences[{}][{}] numeric error".format(i, j)) return embed_question, embed_sentences # RESULTS
def __init__(self, mean=0., logstd=None, std=None, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._mean = tf.convert_to_tensor(mean) warnings.warn("Normal: The order of arguments logstd/std will change " "to std/logstd in the coming version.", FutureWarning) if (logstd is None) == (std is None): raise ValueError("Either std or logstd should be passed but not " "both of them.") elif logstd is None: self._std = tf.convert_to_tensor(std) dtype = assert_same_float_dtype([(self._mean, 'Normal.mean'), (self._std, 'Normal.std')]) logstd = tf.log(self._std) if check_numerics: logstd = tf.check_numerics(logstd, "log(std)") self._logstd = logstd else: # std is None self._logstd = tf.convert_to_tensor(logstd) dtype = assert_same_float_dtype([(self._mean, 'Normal.mean'), (self._logstd, 'Normal.logstd')]) std = tf.exp(self._logstd) if check_numerics: std = tf.check_numerics(std, "exp(logstd)") self._std = std try: tf.broadcast_static_shape(self._mean.get_shape(), self._std.get_shape()) except ValueError: raise ValueError( "mean and std/logstd should be broadcastable to match each " "other. ({} vs. {})".format( self._mean.get_shape(), self._std.get_shape())) self._check_numerics = check_numerics super(Normal, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def __init__(self, mean=0., logstd=None, std=None, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._mean = tf.convert_to_tensor(mean) warnings.warn("FoldNormal: The order of arguments logstd/std will change " "to std/logstd in the coming version.", FutureWarning) if (logstd is None) == (std is None): raise ValueError("Either std or logstd should be passed but not " "both of them.") elif logstd is None: self._std = tf.convert_to_tensor(std) dtype = assert_same_float_dtype([(self._mean, 'FoldNormal.mean'), (self._std, 'FoldNormal.std')]) logstd = tf.log(self._std) if check_numerics: logstd = tf.check_numerics(logstd, "log(std)") self._logstd = logstd else: # std is None self._logstd = tf.convert_to_tensor(logstd) dtype = assert_same_float_dtype([(self._mean, 'FoldNormal.mean'), (self._logstd, 'FoldNormal.logstd')]) std = tf.exp(self._logstd) if check_numerics: std = tf.check_numerics(std, "exp(logstd)") self._std = std try: tf.broadcast_static_shape(self._mean.get_shape(), self._std.get_shape()) except ValueError: raise ValueError( "mean and std/logstd should be broadcastable to match each " "other. ({} vs. {})".format( self._mean.get_shape(), self._std.get_shape())) self._check_numerics = check_numerics super(FoldNormal, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def __init__(self, logits, n_experiments, dtype=None, group_ndims=0, check_numerics=False, **kwargs): self._logits = tf.convert_to_tensor(logits) param_dtype = assert_same_float_dtype( [(self._logits, 'Binomial.logits')]) if dtype is None: dtype = tf.int32 assert_same_float_and_int_dtype([], dtype) sign_err_msg = "n_experiments must be positive" if isinstance(n_experiments, int): if n_experiments <= 0: raise ValueError(sign_err_msg) self._n_experiments = n_experiments else: try: n_experiments = tf.convert_to_tensor(n_experiments, tf.int32) except ValueError: raise TypeError('n_experiments must be int32') _assert_rank_op = tf.assert_rank( n_experiments, 0, message="n_experiments should be a scalar (0-D Tensor).") _assert_positive_op = tf.assert_greater( n_experiments, 0, message=sign_err_msg) with tf.control_dependencies([_assert_rank_op, _assert_positive_op]): self._n_experiments = tf.identity(n_experiments) self._check_numerics = check_numerics super(Binomial, self).__init__( dtype=dtype, param_dtype=param_dtype, is_continuous=False, is_reparameterized=False, group_ndims=group_ndims, **kwargs)
def __init__(self, mean, cov_tril, group_ndims=0, is_reparameterized=True, use_path_derivative=False, check_numerics=False, **kwargs): self._check_numerics = check_numerics self._mean = tf.convert_to_tensor(mean) self._mean = assert_rank_at_least_one( self._mean, 'MultivariateNormalCholesky.mean') self._n_dim = get_shape_at(self._mean, -1) self._cov_tril = tf.convert_to_tensor(cov_tril) self._cov_tril = assert_rank_at_least( self._cov_tril, 2, 'MultivariateNormalCholesky.cov_tril') # Static shape check expected_shape = self._mean.get_shape().concatenate( [self._n_dim if isinstance(self._n_dim, int) else None]) self._cov_tril.get_shape().assert_is_compatible_with(expected_shape) # Dynamic expected_shape = tf.concat( [tf.shape(self._mean), [self._n_dim]], axis=0) actual_shape = tf.shape(self._cov_tril) msg = ['MultivariateNormalCholesky.cov_tril should have compatible ' 'shape with mean. Expected', expected_shape, ' got ', actual_shape] assert_ops = [tf.assert_equal(expected_shape, actual_shape, msg)] with tf.control_dependencies(assert_ops): self._cov_tril = tf.identity(self._cov_tril) dtype = assert_same_float_dtype( [(self._mean, 'MultivariateNormalCholesky.mean'), (self._cov_tril, 'MultivariateNormalCholesky.cov_tril')]) super(MultivariateNormalCholesky, self).__init__( dtype=dtype, param_dtype=dtype, is_continuous=True, is_reparameterized=is_reparameterized, use_path_derivative=use_path_derivative, group_ndims=group_ndims, **kwargs)
def model(self, input_label, output_word, batch_size, vocabulary_size=VOCABULARY_SIZE, embedding_size=EMBEDDINGS_SIZE, num_negative_samples=W2V_NEGATIVE_NUM_SAMPLES, learning_rate_initial=W2V_LEARNING_RATE_INITIAL, learning_rate_decay=W2V_LEARNING_RATE_DECAY, learning_rate_decay_steps=W2V_LEARNING_RATE_DECAY_STEPS): self.global_step = training_util.get_or_create_global_step() # inputs/outputs input_label_reshaped = tf.reshape(input_label, [batch_size]) output_word_reshaped = tf.reshape(output_word, [batch_size, 1]) # embeddings matrix_dimension = [vocabulary_size, embedding_size] self.embeddings = tf.get_variable(shape=matrix_dimension, initializer=layers.xavier_initializer(), dtype=tf.float32, name='embeddings') embed = tf.nn.embedding_lookup(self.embeddings, input_label_reshaped) # NCE loss stddev = 1.0 / math.sqrt(embedding_size) nce_weights = tf.Variable(tf.truncated_normal(matrix_dimension, stddev=stddev)) nce_biases = tf.Variable(tf.zeros([vocabulary_size])) nce_loss = tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=output_word_reshaped, inputs=embed, num_sampled=num_negative_samples, num_classes=vocabulary_size) self.loss = tf.reduce_mean(nce_loss) tf.summary.scalar('loss', self.loss) # learning rate & optimizer self.learning_rate = tf.train.exponential_decay(learning_rate_initial, self.global_step, learning_rate_decay_steps, learning_rate_decay, staircase=True, name='learning_rate') tf.summary.scalar('learning_rate', self.learning_rate) sgd = tf.train.GradientDescentOptimizer(self.learning_rate) self.optimizer = sgd.minimize(self.loss, global_step=self.global_step) # saver to save the model self.saver = tf.train.Saver() # check a nan value in the loss self.loss = tf.check_numerics(self.loss, 'loss is nan') # embeddings config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = self.embeddings.name filename_tsv = '{}_{}.tsv'.format('word2vec_dataset', vocabulary_size) if not os.path.exists(self.log_dir): os.makedirs(self.log_dir) shutil.copy(os.path.join(DIR_DATA_WORD2VEC, filename_tsv), self.log_dir) embedding.metadata_path = filename_tsv summary_writer = tf.summary.FileWriter(self.log_dir) projector.visualize_embeddings(summary_writer, config) # normalize the embeddings to save them norm = tf.sqrt(tf.reduce_sum(tf.square(self.embeddings), 1, keep_dims=True)) self.normalized_embeddings = self.embeddings / norm return None
