Python tensorflow.scalar_mul() Examples

def soft_arg_min(filtered_cost_volume, name):
    with tf.variable_scope(name):
        #input.shape (batch, depth, H, W)
        # softargmin to disp image, outsize of (B, H, W)

        #print('filtered_cost_volume:',filtered_cost_volume.shape)
        probability_volume = tf.nn.softmax(tf.scalar_mul(-1, filtered_cost_volume),
                                           dim=1, name='prob_volume')
        #print('probability_volume:',probability_volume.shape)
        volume_shape = tf.shape(probability_volume)
        soft_1d = tf.cast(tf.range(0, volume_shape[1], dtype=tf.int32),tf.float32)
        soft_4d = tf.tile(soft_1d, tf.stack([volume_shape[0] * volume_shape[2] * volume_shape[3]]))
        soft_4d = tf.reshape(soft_4d, [volume_shape[0], volume_shape[2], volume_shape[3], volume_shape[1]])
        soft_4d = tf.transpose(soft_4d, [0, 3, 1, 2])
        estimated_disp_image = tf.reduce_sum(soft_4d * probability_volume, axis=1)
        #print(estimated_disp_image.shape)
        #estimated_disp_image = tf.expand_dims(estimated_disp_image, axis=3)
        return estimated_disp_image 

def clip_norm(g, c, n):
    if c > 0:
        if K.backend() == 'tensorflow':
            import tensorflow as tf
            import copy
            condition = n >= c
            then_expression = tf.scalar_mul(c / n, g)
            else_expression = g

            if hasattr(then_expression, 'get_shape'):
                g_shape = copy.copy(then_expression.get_shape())
            elif hasattr(then_expression, 'dense_shape'):
                g_shape = copy.copy(then_expression.dense_shape)
            if condition.dtype != tf.bool:
                condition = tf.cast(condition, 'bool')
            g = K.tensorflow_backend.control_flow_ops.cond(
                condition, lambda: then_expression, lambda: else_expression)
            if hasattr(then_expression, 'get_shape'):
                g.set_shape(g_shape)
            elif hasattr(then_expression, 'dense_shape'):
                g._dense_shape = g_shape
        else:
            g = K.switch(n >= c, g * c / n, g)
    return g 

def difference(predicted, target, loss_difference, epsilon=1e-2):
    if loss_difference == LossDifferenceEnum.DIFFERENCE:
      result = tf.subtract(predicted, target)
    elif loss_difference == LossDifferenceEnum.ABSOLUTE:
      difference = tf.subtract(predicted, target)
      result = tf.abs(difference)
    elif loss_difference == LossDifferenceEnum.SMOOTH_ABSOLUTE:
      difference = tf.subtract(predicted, target)
      absolute_difference = tf.abs(difference)
      result = tf.where(
          tf.less(absolute_difference, 1),
          tf.scalar_mul(0.5, tf.square(absolute_difference)),
          tf.subtract(absolute_difference, 0.5))
    elif loss_difference == LossDifferenceEnum.SQUARED:
      result = tf.squared_difference(predicted, target)
    elif loss_difference == LossDifferenceEnum.SMAPE:
      # https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error
      absolute_difference = tf.abs(tf.subtract(predicted, target))
      denominator = tf.add(tf.add(tf.abs(predicted), tf.abs(target)), epsilon)
      result = tf.divide(absolute_difference, denominator)
    result = tf.reduce_sum(result, axis=3)
    return result 

def focal_loss_(labels, pred, anchor_state, alpha=0.25, gamma=2.0):

    # filter out "ignore" anchors
    indices = tf.reshape(tf.where(tf.not_equal(anchor_state, -1)), [-1, ])
    labels = tf.gather(labels, indices)
    pred = tf.gather(pred, indices)

    logits = tf.cast(pred, tf.float32)
    onehot_labels = tf.cast(labels, tf.float32)
    ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=onehot_labels, logits=logits)
    predictions = tf.sigmoid(logits)
    predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1.-predictions)
    alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
    alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1-alpha_t)
    loss = ce * tf.pow(1-predictions_pt, gamma) * alpha_t
    positive_mask = tf.cast(tf.greater(labels, 0), tf.float32)
    return tf.reduce_sum(loss) / tf.maximum(tf.reduce_sum(positive_mask), 1) 

def focal_loss_(labels, pred, anchor_state, alpha=0.25, gamma=2.0):

    # filter out "ignore" anchors
    indices = tf.reshape(tf.where(tf.not_equal(anchor_state, -1)), [-1, ])
    labels = tf.gather(labels, indices)
    pred = tf.gather(pred, indices)

    logits = tf.cast(pred, tf.float32)
    onehot_labels = tf.cast(labels, tf.float32)
    ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=onehot_labels, logits=logits)
    predictions = tf.sigmoid(logits)
    predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1.-predictions)
    alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
    alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1-alpha_t)
    loss = ce * tf.pow(1-predictions_pt, gamma) * alpha_t
    positive_mask = tf.cast(tf.greater(labels, 0), tf.float32)
    return tf.reduce_sum(loss) / tf.maximum(tf.reduce_sum(positive_mask), 1) 

def compute_gradients(self, loss, var_list=None, *args, **kwargs):
        if var_list is None:
            var_list = (
                    tf.trainable_variables() +
                    tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))

        replaced_list = var_list

        if self._scale != 1.0:
            loss = tf.scalar_mul(self._scale, loss)

        gradvar = self._optimizer.compute_gradients(loss, replaced_list, *args, **kwargs)

        final_gradvar = []
        for orig_var, (grad, var) in zip(var_list, gradvar):
            if var is not orig_var:
                grad = tf.cast(grad, orig_var.dtype)
            if self._scale != 1.0:
                grad = tf.scalar_mul(1. / self._scale, grad)
            final_gradvar.append((grad, orig_var))

        return final_gradvar 

def call(self, inputs, **kwargs):
        if K.ndim(inputs) != 3:
            raise ValueError(
                "Unexpected inputs dimensions %d, expect to be 3 dimensions"
                % (K.ndim(inputs)))

        if inputs.shape[1] != self.num_fields:
            raise ValueError("Mismatch in number of fields {} and \
                 concatenated embeddings dims {}".format(self.num_fields, inputs.shape[1]))

        pairwise_inner_prods = []
        for fi, fj in itertools.combinations(range(self.num_fields), 2):
            # get field strength for pair fi and fj
            r_ij = self.field_strengths[fi, fj]

            # get embeddings for the features of both the fields
            feat_embed_i = tf.squeeze(inputs[0:, fi:fi + 1, 0:], axis=1)
            feat_embed_j = tf.squeeze(inputs[0:, fj:fj + 1, 0:], axis=1)

            f = tf.scalar_mul(r_ij, batch_dot(feat_embed_i, feat_embed_j, axes=1))
            pairwise_inner_prods.append(f)

        sum_ = tf.add_n(pairwise_inner_prods)
        return sum_ 

def skip_connection_layer(input_layer, skip_layer, str, is_training=False):
    _, sx, sy, sf = input_layer.outputs.get_shape().as_list()
    _, sx_, sy_, sf_ = skip_layer.outputs.get_shape().as_list()
    
    assert (sx_,sy_,sf_) == (sx,sy,sf)

    # skip-connection domain transformation, from LDR encoder to log HDR decoder
    skip_layer.outputs = tf.log(tf.pow(tf.scalar_mul(1.0/255, skip_layer.outputs), 2.0)+1.0/255.0)

    # specify weights for fusion of concatenation, so that it performs an element-wise addition
    weights = np.zeros((1, 1, sf+sf_, sf))
    for i in range(sf):
        weights[0, 0, i, i] = 1
        weights[:, :, i+sf_, i] = 1
    add_init = tf.constant_initializer(value=weights, dtype=tf.float32)

    # concatenate layers
    network = tl.layers.ConcatLayer([input_layer,skip_layer], concat_dim=3, name ='%s/skip_connection'%str)

    # fuse concatenated layers using the specified weights for initialization
    network = tl.layers.Conv2dLayer(network,
                    act = tf.identity,
                    shape = [1, 1, sf+sf_, sf],
                    strides = [1, 1, 1, 1],
                    padding = 'SAME',
                    W_init = add_init,
                    b_init = tf.constant_initializer(value=0.0),
                    name = str)

    return network


# Deconvolution layer 

def model(x, batch_size=1, is_training=False):

    # Encoder network (VGG16, until pool5)
    x_in = tf.scalar_mul(255.0, x)
    net_in = tl.layers.InputLayer(x_in, name='input_layer')
    conv_layers, skip_layers = encoder(net_in)

    # Fully convolutional layers on top of VGG16 conv layers
    network = tl.layers.Conv2dLayer(conv_layers,
                    act = tf.identity,
                    shape = [3, 3, 512, 512],
                    strides = [1, 1, 1, 1],
                    padding='SAME',
                    name ='encoder/h6/conv')
    network = tl.layers.BatchNormLayer(network, is_train=is_training, name='encoder/h6/batch_norm')
    network.outputs = tf.nn.relu(network.outputs, name='encoder/h6/relu')

    # Decoder network
    network = decoder(network, skip_layers, batch_size, is_training)

    if is_training:
        return network, conv_layers

    return network


# Final prediction of the model, including blending with input 

def mlp_definition(features,nn,num_classes):
  """ Defines layers in tensorflow neural network, using info from nn python structure. """
  # Define input layer, cast data as tensor
  features = features['x']
  layers = [tf.reshape(tf.cast(features,tf.float32), features.shape)]  ### NEED TO VERIFY FLOAT32 

  # Loop over layers and build tensorflow network
  for lidx in range(1,nn.num_internal_layers+1):
    plist = get_layer_parents(nn.conn_mat.viewkeys(),lidx)
    # Define or concatenate parents
    parent_layers = [layers[i] for i in plist]
    input_layer = tf.concat(parent_layers,1)  ### NEED TO VERIFY CONCAT ALONG AXIS 1
    # Get number of hidden units
    num_units = nn.num_units_in_each_layer[lidx]
    if num_units==None: num_units=1
    # Define activation function
    act_str = nn.layer_labels[lidx]
    # define next layer
    layers.append(tf.layers.dense(input_layer,num_units,use_bias=True,activation=activation_dict[act_str]))

  # Define output layer
  plist = get_layer_parents(nn.conn_mat.viewkeys(),lidx+1)
  parent_layers = [layers[i] for i in plist]
  #scalar_mult = tf.Variable(1./(len(plist)+1),tf.float32) ### NEED TO VERIFY FLOAT 32
  scalar_mult = tf.Variable(1./len(plist),tf.float32) ### NEED TO VERIFY FLOAT 32
  input_layer = tf.scalar_mul(scalar_mult,tf.add_n(parent_layers))
  # For regression
  if nn.class_or_reg=='reg':
    op_layer = tf.layers.dense(input_layer,1,use_bias=True,activation=None)
  # For classification
  elif nn.class_or_reg=='class':
    op_layer = tf.layers.dense(input_layer,num_classes,use_bias=True,activation=tf.nn.softmax)
  else:
    pass
  return op_layer 

def mlp_definition(features,nn,num_classes):
  """ Defines layers in tensorflow neural network, using info from nn python structure. """
  # Define input layer, cast data as tensor
  features = features['x']
  layers = [tf.reshape(tf.cast(features,tf.float32), features.shape)]  ### NEED TO VERIFY FLOAT32 

  # Loop over layers and build tensorflow network
  for lidx in range(1,nn.num_internal_layers+1):
    plist = get_layer_parents(nn.conn_mat.viewkeys(),lidx)
    # Define or concatenate parents
    parent_layers = [layers[i] for i in plist]
    input_layer = tf.concat(parent_layers,1)  ### NEED TO VERIFY CONCAT ALONG AXIS 1
    # Get number of hidden units
    num_units = nn.num_units_in_each_layer[lidx]
    if num_units==None: num_units=1
    # Define activation function
    act_str = nn.layer_labels[lidx]
    # define next layer
    layers.append(tf.layers.dense(input_layer,num_units,use_bias=True,activation=activation_dict[act_str]))

  # Define output layer
  plist = get_layer_parents(nn.conn_mat.viewkeys(),lidx+1)
  parent_layers = [layers[i] for i in plist]
  #scalar_mult = tf.Variable(1./(len(plist)+1),tf.float32) ### NEED TO VERIFY FLOAT 32
  scalar_mult = tf.Variable(1./len(plist),tf.float32) ### NEED TO VERIFY FLOAT 32
  input_layer = tf.scalar_mul(scalar_mult,tf.add_n(parent_layers))
  # For regression
  if nn.class_or_reg=='reg':
    op_layer = tf.layers.dense(input_layer,1,use_bias=True,activation=None)
  # For classification
  elif nn.class_or_reg=='class':
    op_layer = tf.layers.dense(input_layer,num_classes,use_bias=True,activation=tf.nn.softmax)
  else:
    pass
  return op_layer 

def batched_sparse_tensor_to_sparse_block_diagonal(batched_sparse_tensor,
                                                   name=None):
    """Constructs a block-diagonal SparseTensor from batched SparseTensor
    `batched_sparse_tensor`. Each block corresponds to a batch of
    `batched_sparse_tensor` with ordering preserved.

    Args:
        batched_sparse_tensor (`SparseTensor`): Input with dense_shape
            [BATCH_SIZE, M, N]
        name: Name for this operation (optional). If not set, defaults to
            "batched_sparse_tensor_to_sparse_block_diagonal".

    Returns:
        (`SparseTensor`) Block diagonal with shape
            [BATCH_SIZE * M, BATCH_SIZE * N].
    """
    with tf.name_scope(name,
                       "batched_sparse_tensor_to_sparse_block_diagonal",
                       [batched_sparse_tensor]):
        # Dense shape of batched_sparse_tensor
        shape = tf.shape(batched_sparse_tensor,
                         out_type=tf.int64,
                         name="batched_sparse_tensor_dense_shape")

        # Calculate block-diagonal indices. The mapping is
        # (batch_num, x, y) -> (batch_num * M + x, batch_num * N + y)
        batch_nums = batched_sparse_tensor.indices[:, 0]
        offsets_x = tf.scalar_mul(shape[1], batch_nums)
        offsets_y = tf.scalar_mul(shape[2], batch_nums)
        new_indices_x = tf.add(offsets_x, batched_sparse_tensor.indices[:, 1])
        new_indices_y = tf.add(offsets_y, batched_sparse_tensor.indices[:, 2])
        indices = tf.stack([new_indices_x, new_indices_y], axis=1)

        values = batched_sparse_tensor.values
        dense_shape = (shape[0] * shape[1], shape[0] * shape[2])

        return tf.SparseTensor(
            indices=indices, values=values, dense_shape=dense_shape) 

def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
        with tf.name_scope("loss_hinge"):
            float_labels = tf.cast(labels, tf.float32)
            all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
            all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
            sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
            hinge_loss = tf.maximum(
              all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
            return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1)) 

def __init__(self, x, y=None, teacherLogits=None, lr=1e-04, nClasses=8, imgXdim=84, imgYdim=84, batchSize=64, keepProb=1.0, temperature=8, lambda_=0.5):
		self.x = x
		self.w = {}
		self.b = {}
		self.y = y 
		self.teacherLogits = teacherLogits
		self.lambda_ = lambda_ 
		self.T = temperature
		self.imgXdim = imgXdim
		self.imgYdim = imgYdim
		self.nClasses = nClasses
		self.batchSize = batchSize 
		self.learningRate = lr 
		self.dropout = keepProb
		self.fcOutSize = 48
		
		# Initialize parameters randomly and run
		self.initParameters()
		self.output, self.layerInfo = self.run() 
		
		if self.teacherLogits != None: # For training
			# Define losses and optimizers & train the architecture with KD 
			self.outputTeacher = tf.scalar_mul(1.0 / self.T, self.teacherLogits)
			self.outputTeacher = tf.nn.softmax(self.outputTeacher)
			self.cost_1 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.output, labels=self.y))
			self.pred = tf.nn.softmax(self.output)
			self.output = tf.scalar_mul(1.0 / self.T, self.output)
			self.cost_2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.output, labels=self.outputTeacher))
			self.cost = ((1.0 - lambda_) * self.cost_1 + lambda_ * self.cost_2)
			self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learningRate).minimize(self.cost)		
		else: # For standalone testing
			if self.y != None:
				self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.output, labels=self.y))
			self.pred = tf.nn.softmax(self.output)

		if self.y != None: # For labeled images
			# Evaluate model 
			self.correct_pred= tf.equal(tf.argmax(self.pred, 1), tf.argmax(self.y, 1))
			self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) 

def AddGaussianNoise(t, sigma, noise_rate, name=None):
  """Add i.i.d. Gaussian noise (0, sigma^2) to every entry of t.
  Args:
    t: the input tensor.
    sigma: the stddev of the Gaussian noise.
    name: optional name.
  Returns:
    the noisy tensor.
  """

  with tf.name_scope(values=[t, sigma], name=name,
                     default_name="add_gaussian_noise") as name:
    noisy_t = t + tf.scalar_mul(noise_rate, tf.random_normal(tf.shape(t), stddev=sigma))
  return noisy_t 

def _compute_update(self, param, grad, state):
    """Compute updates of parameters."""

    # get the learning rate at the current index, if the index
    # is greater than the number of available learning rates,
    # use the last one
    index = tf.minimum(state["itr"], self.max_index)
    learning_rate = tf.gather(self.learning_rates, index)

    # update the parameters: parameter - learning_rate * gradient
    updated_param = param - tf.scalar_mul(learning_rate, grad)

    return updated_param, {"itr": state["itr"] + 1} 

def add_to_sources_dictionary(self, sources, samples_per_pixel, index_tuple, height, width):
    for i in range(len(index_tuple)):
      if self.feature_prediction.load_data:
        index = index_tuple[i]
        sources[Naming.source_feature_name(self.feature_prediction.name, index=i)] = self.source[samples_per_pixel][index]
      else:
        assert self.feature_prediction.feature_prediction_type != FeaturePredictionType.AUXILIARY
        source = tf.ones([height, width, self.feature_prediction.number_of_channels])
        if self.feature_prediction.feature_prediction_type != FeaturePredictionType.COLOR:
          # Direct and indirect need to be 0.5.
          source = tf.scalar_mul(0.5, source)
        sources[Naming.source_feature_name(self.feature_prediction.name, index=i)] = source 

def _compute_update(self, param, grad, state):
    return param - tf.scalar_mul(self.learning_rate, grad), state 

def loss(self):
    result = 0.
    
    # Allow to have the loss for multiple prediction scales.
    scale_index_count = 1
    if self.use_multiscale_loss:
      scale_index_count = len(self.target)
    
    # Precalculate the common factor for the loss at the different scales.
    scale_weight_factor = 0.
    for scale_index in range(scale_index_count):
      scale_weight_factor = scale_weight_factor + (1. / (4. ** scale_index))
    scale_weight_factor = 1. / scale_weight_factor
    
    with tf.name_scope(Naming.tensorboard_name(self.name + ' Weighted Means')):
      for scale_index in range(scale_index_count):
        scale_factor = scale_weight_factor / (4. ** scale_index)
        if self.mean_weight > 0.:
          result = tf.add(result, tf.scalar_mul(self.mean_weight * scale_factor, self.mean(scale_index)))
        if self.variation_weight > 0.:
          result = tf.add(result, tf.scalar_mul(self.variation_weight * scale_factor, self.variation_mean(scale_index)))
      if self.ms_ssim_weight > 0.:
        result = tf.add(result, tf.scalar_mul(self.ms_ssim_weight, self.ms_ssim()))
    
    with tf.name_scope(Naming.tensorboard_name(self.name + ' Weighted Masked Means')):
      for scale_index in range(scale_index_count):
        scale_factor = scale_weight_factor / (4. ** scale_index)
        if self.masked_mean_weight > 0.:
          result = tf.add(result, tf.scalar_mul(self.masked_mean_weight * scale_factor, self.masked_mean(scale_index)))
        if self.masked_variation_weight > 0.:
          result = tf.add(result, tf.scalar_mul(self.masked_variation_weight * scale_factor, self.masked_variation_mean(scale_index)))
      if self.masked_ms_ssim_weight > 0.:
        result = tf.add(result, tf.scalar_mul(self.masked_ms_ssim_weight, self.masked_ms_ssim()))
    return result 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g 

def apply_gradients(self, gradvars, *args, **kwargs):
        v_list = [tf.norm(tensor=v, ord=2) for _, v in gradvars]
        g_list = [tf.norm(tensor=g, ord=2) if g is not None else 0.0
                  for g, _ in gradvars]
        v_norms = tf.stack(v_list)
        g_norms = tf.stack(g_list)
        zeds = tf.zeros_like(v_norms)
        # assign epsilon if weights or grads = 0, to avoid division by zero
        # also prevent biases to get stuck at initialization (0.)
        cond = tf.logical_and(
            tf.not_equal(v_norms, zeds),
            tf.not_equal(g_norms, zeds))
        true_vals = tf.scalar_mul(self._eta, tf.div(v_norms, g_norms))
        # true_vals = tf.scalar_mul(tf.cast(self._eta, tf.float32), tf.div(tf.cast(v_norms, tf.float32), tf.cast(g_norms, tf.float32)))
        false_vals = tf.fill(tf.shape(v_norms), self._epsilon)
        larc_local_lr = tf.where(cond, true_vals, false_vals)
        if self._clip:
            ones = tf.ones_like(v_norms)
            lr = tf.fill(tf.shape(v_norms), self._learning_rate)
            # We need gradients to compute local learning rate,
            # so compute_gradients from initial optimizer have to called
            # for which learning rate is already fixed
            # We then have to scale the gradients instead of the learning rate.
            larc_local_lr = tf.minimum(tf.div(larc_local_lr, lr), ones)
        gradvars = [(tf.multiply(larc_local_lr[i], g), v)
                    if g is not None else (None, v)
                    for i, (g, v) in enumerate(gradvars)]
        return self._optimizer.apply_gradients(gradvars, *args, **kwargs) 

def dsc_loss(scores, labels):
    scores = tf.sigmoid(scores)
    inter = tf.scalar_mul(2., tf.reduce_sum(tf.multiply(scores, labels), [1, 2, 3]))
    union = tf.add(tf.reduce_sum(scores, [1, 2, 3]), tf.reduce_sum(labels, [1, 2, 3]))
    dsc_loss = tf.reduce_mean(tf.sub(1., tf.div(inter, union)))

    return dsc_loss 

def losses(self, logits, labels, end_points):
        """Default cross entropy losses for image classification.
        """
        with tf.name_scope('xentropy'):
            losses = {}
            # Usual cross entropy
            cross_entropy = tfx.losses.sparse_softmax_cross_entropy(
                logits=logits, labels=labels,
                label_smoothing=self.label_smoothing,
                weights=1.0)
            loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
            losses['cross_entropy'] = loss

            # Auxillary cross entropy.
            if end_points and end_points.get('AuxLogits') is not None:
                aux_logits = end_points['AuxLogits']
                with tf.name_scope('aux_xentropy'):
                    aux_cross_entropy = tfx.losses.sparse_softmax_cross_entropy(
                        logits=aux_logits, labels=labels,
                        label_smoothing=self.label_smoothing)
                    aux_loss = tf.scalar_mul(
                        self.aux_loss_weight,
                        tf.reduce_mean(aux_cross_entropy, name='aux_loss'))
                    losses['aux_cross_entropy'] = aux_loss
                    loss = tf.add_n([loss, aux_loss])
            return loss, losses 

def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
    with tf.name_scope("loss_hinge"):
      float_labels = tf.cast(labels, tf.float32)
      all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
      all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
      sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
      hinge_loss = tf.maximum(
          all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
      return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1))