Python tensorflow.scalar_mul() Examples

The following are 30 code examples of tensorflow.scalar_mul(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module tensorflow , or try the search function .
Example #1
Source File: losses.py    From R3Det_Tensorflow with MIT License 6 votes vote down vote up
def focal_loss_(labels, pred, anchor_state, alpha=0.25, gamma=2.0):

    # filter out "ignore" anchors
    indices = tf.reshape(tf.where(tf.not_equal(anchor_state, -1)), [-1, ])
    labels = tf.gather(labels, indices)
    pred = tf.gather(pred, indices)

    logits = tf.cast(pred, tf.float32)
    onehot_labels = tf.cast(labels, tf.float32)
    ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=onehot_labels, logits=logits)
    predictions = tf.sigmoid(logits)
    predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1.-predictions)
    alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
    alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1-alpha_t)
    loss = ce * tf.pow(1-predictions_pt, gamma) * alpha_t
    positive_mask = tf.cast(tf.greater(labels, 0), tf.float32)
    return tf.reduce_sum(loss) / tf.maximum(tf.reduce_sum(positive_mask), 1) 
Example #2
Source File: keras_patches.py    From keras-image-captioning with MIT License 6 votes vote down vote up
def clip_norm(g, c, n):
    if c > 0:
        if K.backend() == 'tensorflow':
            import tensorflow as tf
            import copy
            condition = n >= c
            then_expression = tf.scalar_mul(c / n, g)
            else_expression = g

            if hasattr(then_expression, 'get_shape'):
                g_shape = copy.copy(then_expression.get_shape())
            elif hasattr(then_expression, 'dense_shape'):
                g_shape = copy.copy(then_expression.dense_shape)
            if condition.dtype != tf.bool:
                condition = tf.cast(condition, 'bool')
            g = K.tensorflow_backend.control_flow_ops.cond(
                condition, lambda: then_expression, lambda: else_expression)
            if hasattr(then_expression, 'get_shape'):
                g.set_shape(g_shape)
            elif hasattr(then_expression, 'dense_shape'):
                g._dense_shape = g_shape
        else:
            g = K.switch(n >= c, g * c / n, g)
    return g 
Example #3
Source File: losses.py    From RetinaNet_Tensorflow_Rotation with MIT License 6 votes vote down vote up
def focal_loss_(labels, pred, anchor_state, alpha=0.25, gamma=2.0):

    # filter out "ignore" anchors
    indices = tf.reshape(tf.where(tf.not_equal(anchor_state, -1)), [-1, ])
    labels = tf.gather(labels, indices)
    pred = tf.gather(pred, indices)

    logits = tf.cast(pred, tf.float32)
    onehot_labels = tf.cast(labels, tf.float32)
    ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=onehot_labels, logits=logits)
    predictions = tf.sigmoid(logits)
    predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1.-predictions)
    alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
    alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1-alpha_t)
    loss = ce * tf.pow(1-predictions_pt, gamma) * alpha_t
    positive_mask = tf.cast(tf.greater(labels, 0), tf.float32)
    return tf.reduce_sum(loss) / tf.maximum(tf.reduce_sum(positive_mask), 1) 
Example #4
Source File: train_imagenet_resnet_hvd.py    From sagemaker-tensorflow-training-toolkit with Apache License 2.0 6 votes vote down vote up
def compute_gradients(self, loss, var_list=None, *args, **kwargs):
        if var_list is None:
            var_list = (
                    tf.trainable_variables() +
                    tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))

        replaced_list = var_list

        if self._scale != 1.0:
            loss = tf.scalar_mul(self._scale, loss)

        gradvar = self._optimizer.compute_gradients(loss, replaced_list, *args, **kwargs)

        final_gradvar = []
        for orig_var, (grad, var) in zip(var_list, gradvar):
            if var is not orig_var:
                grad = tf.cast(grad, orig_var.dtype)
            if self._scale != 1.0:
                grad = tf.scalar_mul(1. / self._scale, grad)
            final_gradvar.append((grad, orig_var))

        return final_gradvar 
Example #5
Source File: LossDifference.py    From DeepDenoiser with Apache License 2.0 6 votes vote down vote up
def difference(predicted, target, loss_difference, epsilon=1e-2):
    if loss_difference == LossDifferenceEnum.DIFFERENCE:
      result = tf.subtract(predicted, target)
    elif loss_difference == LossDifferenceEnum.ABSOLUTE:
      difference = tf.subtract(predicted, target)
      result = tf.abs(difference)
    elif loss_difference == LossDifferenceEnum.SMOOTH_ABSOLUTE:
      difference = tf.subtract(predicted, target)
      absolute_difference = tf.abs(difference)
      result = tf.where(
          tf.less(absolute_difference, 1),
          tf.scalar_mul(0.5, tf.square(absolute_difference)),
          tf.subtract(absolute_difference, 0.5))
    elif loss_difference == LossDifferenceEnum.SQUARED:
      result = tf.squared_difference(predicted, target)
    elif loss_difference == LossDifferenceEnum.SMAPE:
      # https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error
      absolute_difference = tf.abs(tf.subtract(predicted, target))
      denominator = tf.add(tf.add(tf.abs(predicted), tf.abs(target)), epsilon)
      result = tf.divide(absolute_difference, denominator)
    result = tf.reduce_sum(result, axis=3)
    return result 
Example #6
Source File: interaction.py    From DeepCTR with Apache License 2.0 6 votes vote down vote up
def call(self, inputs, **kwargs):
        if K.ndim(inputs) != 3:
            raise ValueError(
                "Unexpected inputs dimensions %d, expect to be 3 dimensions"
                % (K.ndim(inputs)))

        if inputs.shape[1] != self.num_fields:
            raise ValueError("Mismatch in number of fields {} and \
                 concatenated embeddings dims {}".format(self.num_fields, inputs.shape[1]))

        pairwise_inner_prods = []
        for fi, fj in itertools.combinations(range(self.num_fields), 2):
            # get field strength for pair fi and fj
            r_ij = self.field_strengths[fi, fj]

            # get embeddings for the features of both the fields
            feat_embed_i = tf.squeeze(inputs[0:, fi:fi + 1, 0:], axis=1)
            feat_embed_j = tf.squeeze(inputs[0:, fj:fj + 1, 0:], axis=1)

            f = tf.scalar_mul(r_ij, batch_dot(feat_embed_i, feat_embed_j, axes=1))
            pairwise_inner_prods.append(f)

        sum_ = tf.add_n(pairwise_inner_prods)
        return sum_ 
Example #7
Source File: utils.py    From PSMNet-Tensorflow with MIT License 6 votes vote down vote up
def soft_arg_min(filtered_cost_volume, name):
    with tf.variable_scope(name):
        #input.shape (batch, depth, H, W)
        # softargmin to disp image, outsize of (B, H, W)

        #print('filtered_cost_volume:',filtered_cost_volume.shape)
        probability_volume = tf.nn.softmax(tf.scalar_mul(-1, filtered_cost_volume),
                                           dim=1, name='prob_volume')
        #print('probability_volume:',probability_volume.shape)
        volume_shape = tf.shape(probability_volume)
        soft_1d = tf.cast(tf.range(0, volume_shape[1], dtype=tf.int32),tf.float32)
        soft_4d = tf.tile(soft_1d, tf.stack([volume_shape[0] * volume_shape[2] * volume_shape[3]]))
        soft_4d = tf.reshape(soft_4d, [volume_shape[0], volume_shape[2], volume_shape[3], volume_shape[1]])
        soft_4d = tf.transpose(soft_4d, [0, 3, 1, 2])
        estimated_disp_image = tf.reduce_sum(soft_4d * probability_volume, axis=1)
        #print(estimated_disp_image.shape)
        #estimated_disp_image = tf.expand_dims(estimated_disp_image, axis=3)
        return estimated_disp_image 
Example #8
Source File: ops.py    From TEGAN with Apache License 2.0 5 votes vote down vote up
def ddx(inpt, channel, dx, scope='ddx', name=None):
    inpt_shape = inpt.get_shape().as_list()
    var = tf.expand_dims( inpt[:,:,:,:,channel], axis=4 )

    with tf.variable_scope(scope):
        ddx1D = tf.constant([-1./60., 3./20., -3./4., 0., 3./4., -3./20., 1./60.], dtype=tf.float32)
        ddx3D = tf.reshape(ddx1D, shape=(-1,1,1,1,1))

    strides = [1,1,1,1,1]
    var_pad = periodic_padding( var, ((3,3),(0,0),(0,0)) )
    output = tf.nn.conv3d(var_pad, ddx3D, strides, padding = 'VALID',
                          data_format = 'NDHWC', name=name)
    output = tf.scalar_mul(1./dx, output)
    
    return output 
Example #9
Source File: ops.py    From TEGAN with Apache License 2.0 5 votes vote down vote up
def ddy(inpt, channel, dy, scope='ddy', name=None):
    inpt_shape = inpt.get_shape().as_list()
    var = tf.expand_dims( inpt[:,:,:,:,channel], axis=4 )

    with tf.variable_scope(scope):
        ddy1D = tf.constant([-1./60., 3./20., -3./4., 0., 3./4., -3./20., 1./60.], dtype=tf.float32)
        ddy3D = tf.reshape(ddy1D, shape=(1,-1,1,1,1))

    strides = [1,1,1,1,1]
    var_pad = periodic_padding( var, ((0,0),(3,3),(0,0)) )
    output = tf.nn.conv3d(var_pad, ddy3D, strides, padding = 'VALID',
                          data_format = 'NDHWC', name=name)
    output = tf.scalar_mul(1./dy, output)
    
    return output 
Example #10
Source File: ops.py    From TEGAN with Apache License 2.0 5 votes vote down vote up
def ddz(inpt, channel, dz, scope='ddz', name=None):
    inpt_shape = inpt.get_shape().as_list()
    var = tf.expand_dims( inpt[:,:,:,:,channel], axis=4 )

    with tf.variable_scope(scope):
        ddz1D = tf.constant([-1./60., 3./20., -3./4., 0., 3./4., -3./20., 1./60.], dtype=tf.float32)
        ddz3D = tf.reshape(ddz1D, shape=(1,1,-1,1,1))

    strides = [1,1,1,1,1]
    var_pad = periodic_padding( var, ((0,0),(0,0),(3,3)) )
    output = tf.nn.conv3d(var_pad, ddz3D, strides, padding = 'VALID',
                          data_format = 'NDHWC', name=name)
    output = tf.scalar_mul(1./dz, output)
    
    return output 
Example #11
Source File: ops.py    From TEGAN with Apache License 2.0 5 votes vote down vote up
def d2dx2(inpt, channel, dx, scope='d2dx2', name=None):
    inpt_shape = inpt.get_shape().as_list()
    var = tf.expand_dims( inpt[:,:,:,:,channel], axis=4 )

    with tf.variable_scope(scope):
        ddx1D = tf.constant([1./90., -3./20., 3./2., -49./18., 3./2., -3./20., 1./90.], dtype=tf.float32)
        ddx3D = tf.reshape(ddx1D, shape=(-1,1,1,1,1))

    strides = [1,1,1,1,1]
    var_pad = periodic_padding( var, ((3,3),(0,0),(0,0)) )
    output = tf.nn.conv3d(var_pad, ddx3D, strides, padding = 'VALID',
                          data_format = 'NDHWC', name=name)
    output = tf.scalar_mul(1./dx**2, output)
    
    return output 
Example #12
Source File: ops.py    From TEGAN with Apache License 2.0 5 votes vote down vote up
def d2dy2(inpt, channel, dy, scope='d2dy2', name=None):
    inpt_shape = inpt.get_shape().as_list()
    var = tf.expand_dims( inpt[:,:,:,:,channel], axis=4 )

    with tf.variable_scope(scope):
        ddy1D = tf.constant([1./90., -3./20., 3./2., -49./18., 3./2., -3./20., 1./90.], dtype=tf.float32)
        ddy3D = tf.reshape(ddy1D, shape=(1,-1,1,1,1))

    strides = [1,1,1,1,1]
    var_pad = periodic_padding( var, ((0,0),(3,3),(0,0)) )
    output = tf.nn.conv3d(var_pad, ddy3D, strides, padding = 'VALID',
                          data_format = 'NDHWC', name=name)
    output = tf.scalar_mul(1./dy**2, output)
    
    return output 
Example #13
Source File: ssd_augmenter.py    From lambda-deep-learning-demo with Apache License 2.0 5 votes vote down vote up
def preprocess_for_eval(image,
                        classes,
                        boxes,
                        resolution,
                        speed_mode=False):
  if speed_mode:
    pass
  else:

    # mean subtraction   
    means = [_R_MEAN, _G_MEAN, _B_MEAN]

    channels = tf.split(axis=2, num_or_size_splits=3, value=image)
    for i in range(3):
      channels[i] -= means[i]

    # image = tf.concat(axis=2, values=channels)

    # caffe swaps color channels
    image = tf.concat(axis=2, values=[channels[2], channels[1], channels[0]]) 


    image, scale, translation = bilinear_resize(image, resolution, depth=3, resize_mode="bilinear")
    # Need this to make later tensor unstack working
    image.set_shape([resolution, resolution, 3])
    x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1)
    x1 = tf.scalar_mul(scale[1], x1)
    y1 = tf.scalar_mul(scale[0], y1)
    x2 = tf.scalar_mul(scale[1], x2)
    y2 = tf.scalar_mul(scale[0], y2)
    boxes = tf.concat([tf.expand_dims(x1, -1),
                       tf.expand_dims(y1, -1),
                       tf.expand_dims(x2, -1),
                       tf.expand_dims(y2, -1)], axis=1)

    boxes = boxes + [translation[1], translation[0], translation[1], translation[0]]


  return image, classes, boxes, scale, translation 
Example #14
Source File: network.py    From RetinaNet with MIT License 5 votes vote down vote up
def focal_loss(self, pred, label, num_class=21, alpha=0.25, gamma=2.0):
        label_one_hot = tf.one_hot(indices=label, depth=num_class, on_value=1.0, off_value=0.0, axis=-1, dtype=tf.float32)
        pt = tf.reduce_sum(tf.multiply(pred, label_one_hot), axis=1)

        gamma_tf = tf.scalar_mul(gamma, tf.ones_like(pt, tf.float32))
        alpha_tf = tf.map_fn(lambda x: 1.0 - alpha if x == 0 else alpha, label, dtype=tf.float32)

        cls_loss = alpha_tf*(-1.0 * tf.pow(1 - pt, gamma_tf) * tf.log(pt))
        #cls_loss = -1.0 * tf.log(pt)
        #cls_loss = -1 * tf.multiply(alpha, tf.multiply(tf.pow(1 - pt, gamma), tf.log(pt)))
        return cls_loss 
Example #15
Source File: network.py    From RetinaNet with MIT License 5 votes vote down vote up
def focal_loss_sigmoid(self, pred, label, num_class=20, alpha=0.25, gamma=2.0):
        label_one_hot = tf.one_hot(indices=label - 1, depth=num_class, on_value=1.0, off_value=0.0, axis=-1, dtype=tf.float32)
        pt = tf.where(tf.equal(label_one_hot, 1.0), pred, 1.0 - pred)

        gamma_tf = tf.scalar_mul(gamma, tf.ones_like(pt, tf.float32))
        #alpha_tf = tf.map_fn(lambda x: 1.0 - alpha if x == 0 else alpha, label_one_hot, dtype=tf.float32)
        alpha_tf = tf.scalar_mul(alpha, tf.ones_like(pt, tf.float32))
        alpha_tf = tf.where(tf.equal(label_one_hot, 1.0), alpha_tf, 1.0 - alpha_tf)
        #alpha_tf = 1.0

        cls_loss = alpha_tf*(-1.0 * tf.pow(1 - pt, gamma_tf) * tf.log(pt))

        return cls_loss 
Example #16
Source File: losses.py    From youtube-8m with Apache License 2.0 5 votes vote down vote up
def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
    with tf.name_scope("loss_hinge"):
      float_labels = tf.cast(labels, tf.float32)
      all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
      all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
      sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
      hinge_loss = tf.maximum(
          all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
      return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1)) 
Example #17
Source File: losses.py    From youtube-8m with Apache License 2.0 5 votes vote down vote up
def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
    with tf.name_scope("loss_hinge"):
      float_labels = tf.cast(labels, tf.float32)
      all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
      all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
      sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
      hinge_loss = tf.maximum(
          all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
      return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1)) 
Example #18
Source File: absorption.py    From graphics with Apache License 2.0 5 votes vote down vote up
def render(voxels, absorption_factor=0.1, cell_size=1.0, name=None):
  """Renders a voxel grid using the absorption-only model, as described in ["Escaping Plato's Cave: 3D Shape From Adversarial Rendering" (Henzler 2019)](https://github.com/henzler/platonicgan).

  Note:
    In the following, A1 to An are optional batch dimensions.

  Args:
    voxels: A tensor of shape `[A1, ..., An, Vx, Vy, Vz, Vd]`, where Vx, Vy, Vz
      are the dimensions of the voxel grid and Vd the dimension of the
      information stored in each voxel (e.g. 3 for RGB color).
    absorption_factor: A scalar representing the density of the volume.
    cell_size: A scalar representing the size of a cell.
    name: A name for this op. Defaults to "absorption_render".

  Returns:
    A tensor of shape `[A1, ..., An, Vx, Vy, Vd]` representing images of size
    (Vx,Vy).

  Raises:
    ValueError: If the shape of the input tensors are not supported.
  """
  with tf.compat.v1.name_scope(name, "absorption_render", [voxels]):
    voxels = tf.convert_to_tensor(value=voxels)

    shape.check_static(
        tensor=voxels, tensor_name="voxels", has_rank_greater_than=3)

    transmission = tf.scalar_mul(absorption_factor / cell_size, voxels)
    transmission = tf.ones_like(transmission) - transmission
    transmission = tf.clip_by_value(
        transmission, clip_value_min=1e-6, clip_value_max=1.0)

    image = tf.math.log(transmission)
    image = tf.reduce_sum(input_tensor=image, axis=-2)
    image = tf.math.exp(image)
    image = tf.ones_like(image) - image
    return image


# API contains all public functions and classes. 
Example #19
Source File: ops.py    From tfdeploy with MIT License 5 votes vote down vote up
def test_scalar_mul(self):
        t = tf.scalar_mul(1, tf.Variable(self.random(3, 5)))
        self.check(t) 
Example #20
Source File: fisher_blocks.py    From kfac with Apache License 2.0 5 votes vote down vote up
def _multiply_factored_matrix(self, left_factor, right_factor, vector,
                                extra_scale=1.0, transpose_left=False,
                                transpose_right=False):
    """Multiplies a factored matrix."""
    reshaped_vector = utils.layer_params_to_mat2d(vector)
    reshaped_out = right_factor.matmul_right(reshaped_vector,
                                             adjoint=transpose_right)
    reshaped_out = left_factor.matmul(reshaped_out,
                                      adjoint=transpose_left)
    if extra_scale != 1.0:
      reshaped_out = tf.scalar_mul(extra_scale, reshaped_out)
    return utils.mat2d_to_layer_params(vector, reshaped_out) 
Example #21
Source File: learning_rate_schedule.py    From yolo_v2 with Apache License 2.0 5 votes vote down vote up
def _compute_update(self, param, grad, state):
    """Compute updates of parameters."""

    # get the learning rate at the current index, if the index
    # is greater than the number of available learning rates,
    # use the last one
    index = tf.minimum(state["itr"], self.max_index)
    learning_rate = tf.gather(self.learning_rates, index)

    # update the parameters: parameter - learning_rate * gradient
    updated_param = param - tf.scalar_mul(learning_rate, grad)

    return updated_param, {"itr": state["itr"] + 1} 
Example #22
Source File: global_learning_rate.py    From yolo_v2 with Apache License 2.0 5 votes vote down vote up
def _compute_update(self, param, grad, state):
    return param - tf.scalar_mul(self.learning_rate, grad), state 
Example #23
Source File: variable_mgr_util.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up
def aggregate_single_gradient_using_copy(grad_and_vars, use_mean,
                                         check_inf_nan):
  """Calculate the average gradient for a shared variable across all towers.

  Note that this function provides a synchronization point across all towers.

  Args:
    grad_and_vars: A list or tuple of (gradient, variable) tuples. Each
      (gradient, variable) pair within the outer list represents the gradient
      of the variable calculated for a single tower, and the number of pairs
      equals the number of towers.
    use_mean: if True, mean is taken, else sum of gradients is taken.
    check_inf_nan: check grads for nans and infs.

  Returns:
    The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
      gradient has been averaged across all towers. The variable is chosen from
      the first tower. The has_nan_or_inf indicates the grads has nan or inf.
  """
  grads = [g for g, _ in grad_and_vars]
  if any(isinstance(g, tf.IndexedSlices) for g in grads):
    # TODO(reedwm): All-reduce IndexedSlices more effectively.
    grad = gradients_impl._AggregateIndexedSlicesGradients(grads)  # pylint: disable=protected-access
  else:
    grad = tf.add_n(grads)

  if use_mean and len(grads) > 1:
    grad = tf.scalar_mul(1.0 / len(grads), grad)

  v = grad_and_vars[0][1]
  if check_inf_nan:
    with tf.name_scope('check_for_inf_and_nan'):
      has_nan_or_inf = tf.logical_not(tf.reduce_all(tf.is_finite(grads)))
    return (grad, v), has_nan_or_inf
  else:
    return (grad, v), None 
Example #24
Source File: variable_mgr_util.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up
def aggregate_single_gradient_using_copy(grad_and_vars, use_mean,
                                         check_inf_nan):
  """Calculate the average gradient for a shared variable across all towers.

  Note that this function provides a synchronization point across all towers.

  Args:
    grad_and_vars: A list or tuple of (gradient, variable) tuples. Each
      (gradient, variable) pair within the outer list represents the gradient
      of the variable calculated for a single tower, and the number of pairs
      equals the number of towers.
    use_mean: if True, mean is taken, else sum of gradients is taken.
    check_inf_nan: check grads for nans and infs.

  Returns:
    The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
      gradient has been averaged across all towers. The variable is chosen from
      the first tower. The has_nan_or_inf indicates the grads has nan or inf.
  """
  grads = [g for g, _ in grad_and_vars]
  if any(isinstance(g, tf.IndexedSlices) for g in grads):
    # TODO(reedwm): All-reduce IndexedSlices more effectively.
    grad = gradients_impl._AggregateIndexedSlicesGradients(grads)  # pylint: disable=protected-access
  else:
    grad = tf.add_n(grads)

  if use_mean and len(grads) > 1:
    grad = tf.scalar_mul(1.0 / len(grads), grad)

  v = grad_and_vars[0][1]
  if check_inf_nan:
    with tf.name_scope('check_for_inf_and_nan'):
      has_nan_or_inf = tf.logical_not(tf.reduce_all(tf.is_finite(grads)))
    return (grad, v), has_nan_or_inf
  else:
    return (grad, v), None 
Example #25
Source File: variable_mgr_util.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up
def aggregate_single_gradient_using_copy(grad_and_vars, use_mean,
                                         check_inf_nan):
  """Calculate the average gradient for a shared variable across all towers.

  Note that this function provides a synchronization point across all towers.

  Args:
    grad_and_vars: A list or tuple of (gradient, variable) tuples. Each
      (gradient, variable) pair within the outer list represents the gradient
      of the variable calculated for a single tower, and the number of pairs
      equals the number of towers.
    use_mean: if True, mean is taken, else sum of gradients is taken.
    check_inf_nan: check grads for nans and infs.

  Returns:
    The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
      gradient has been averaged across all towers. The variable is chosen from
      the first tower. The has_nan_or_inf indicates the grads has nan or inf.
  """
  grads = [g for g, _ in grad_and_vars]
  if any(isinstance(g, tf.IndexedSlices) for g in grads):
    # TODO(reedwm): All-reduce IndexedSlices more effectively.
    grad = gradients_impl._AggregateIndexedSlicesGradients(grads)  # pylint: disable=protected-access
  else:
    grad = tf.add_n(grads)

  if use_mean and len(grads) > 1:
    grad = tf.scalar_mul(1.0 / len(grads), grad)

  v = grad_and_vars[0][1]
  if check_inf_nan:
    with tf.name_scope('check_for_inf_and_nan'):
      has_nan_or_inf = tf.logical_not(tf.reduce_all(tf.is_finite(grads)))
    return (grad, v), has_nan_or_inf
  else:
    return (grad, v), None 
Example #26
Source File: hc_deeplab.py    From SIGGRAPH18SSS with MIT License 5 votes vote down vote up
def lossfunction(self, tweightmat, tindicator, tembeddings):

		with tf.variable_scope('loss_computation') as scope:
			# tembeddings: #pts x 64
			sqrvals = tf.reduce_sum(tf.square(tembeddings), 1, keep_dims=True)
			# sqrvals: #pts x 1
			sqrvalsmat = tf.tile(sqrvals, [1, tf.shape(sqrvals)[0]])
			sqrvalsmat2 = tf.add(sqrvalsmat,tf.transpose(sqrvalsmat))
			distmat =  tf.add(sqrvalsmat2, tf.scalar_mul(-2.0, tf.matmul(tembeddings,  tf.transpose(tembeddings))))/64.0

			sigmamat = tf.scalar_mul(2.0, tf.reciprocal(1.0+tf.exp(distmat)))
			posnegmapping = tf.log(tf.add(tf.scalar_mul(0.5, 1.0-tindicator), tf.multiply(tindicator, sigmamat)))
			wcrossentropy = tf.multiply(tf.negative(tindicator+2.0), posnegmapping)
			lossval = tf.reduce_mean(wcrossentropy)
		return lossval 
Example #27
Source File: optimizers.py    From GraphicDesignPatternByPython with MIT License 5 votes vote down vote up
def clip_norm(g, c, n):
    """Clip the gradient `g` if the L2 norm `n` exceeds `c`.

    # Arguments
        g: Tensor, the gradient tensor
        c: float >= 0. Gradients will be clipped
            when their L2 norm exceeds this value.
        n: Tensor, actual norm of `g`.

    # Returns
        Tensor, the gradient clipped if required.
    """
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 
Example #28
Source File: learning_rate_schedule.py    From Gun-Detector with Apache License 2.0 5 votes vote down vote up
def _compute_update(self, param, grad, state):
    """Compute updates of parameters."""

    # get the learning rate at the current index, if the index
    # is greater than the number of available learning rates,
    # use the last one
    index = tf.minimum(state["itr"], self.max_index)
    learning_rate = tf.gather(self.learning_rates, index)

    # update the parameters: parameter - learning_rate * gradient
    updated_param = param - tf.scalar_mul(learning_rate, grad)

    return updated_param, {"itr": state["itr"] + 1} 
Example #29
Source File: global_learning_rate.py    From Gun-Detector with Apache License 2.0 5 votes vote down vote up
def _compute_update(self, param, grad, state):
    return param - tf.scalar_mul(self.learning_rate, grad), state 
Example #30
Source File: losses.py    From youtube-8m with Apache License 2.0 5 votes vote down vote up
def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
    with tf.name_scope("loss_hinge"):
      float_labels = tf.cast(labels, tf.float32)
      all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
      all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
      sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
      hinge_loss = tf.maximum(
          all_zeros,
          tf.scalar_mul(b, all_ones) - sign_labels * predictions)
      return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1))