Python keras.backend.sqrt() Examples

The following are code examples for showing how to use keras.backend.sqrt(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: Kaggle-Statoil-Challenge   Author: adodd202   File: capsulelayers.py    MIT License 6 votes vote down vote up
def call(self, inputs, **kwargs):
        if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.
            assert len(inputs) == 2
            inputs, mask = inputs
        else:  # if no true label, mask by the max length of capsules. Mainly used for prediction
            # compute lengths of capsules
            x = K.sqrt(K.sum(K.square(inputs), -1))
            # generate the mask which is a one-hot code.
            # mask.shape=[None, n_classes]=[None, num_capsule]
            mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])

        # inputs.shape=[None, num_capsule, dim_capsule]
        # mask.shape=[None, num_capsule]
        # masked.shape=[None, num_capsule * dim_capsule]
        masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))
        return masked 
Example 2
Project: deep-learning-note   Author: wdxtub   File: 7_visualize_filters.py    MIT License 6 votes vote down vote up
def generate_pattern(layer_name, filter_index, size=150):
    # 过滤器可视化函数
    layer_output = model.get_layer(layer_name).output
    loss = K.mean(layer_output[:, :, :, filter_index])
    grads = K.gradients(loss, model.input)[0]
    grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
    iterate = K.function([model.input], [loss, grads])
    input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.
    
    step = 1
    for _ in range(40):
        loss_value, grads_value = iterate([input_img_data])
        input_img_data += grads_value * step
    
    img = input_img_data[0]
    return deprocess_image(img) 
Example 3
Project: Keras-GAN   Author: eriklindernoren   File: wgan_gp.py    MIT License 6 votes vote down vote up
def gradient_penalty_loss(self, y_true, y_pred, averaged_samples):
        """
        Computes gradient penalty based on prediction and weighted real / fake samples
        """
        gradients = K.gradients(y_pred, averaged_samples)[0]
        # compute the euclidean norm by squaring ...
        gradients_sqr = K.square(gradients)
        #   ... summing over the rows ...
        gradients_sqr_sum = K.sum(gradients_sqr,
                                  axis=np.arange(1, len(gradients_sqr.shape)))
        #   ... and sqrt
        gradient_l2_norm = K.sqrt(gradients_sqr_sum)
        # compute lambda * (1 - ||grad||)^2 still for each single sample
        gradient_penalty = K.square(1 - gradient_l2_norm)
        # return the mean as loss over all the batch samples
        return K.mean(gradient_penalty) 
Example 4
Project: deepflying   Author: dslab-deepflying   File: wgan_gp.py    GNU General Public License v3.0 6 votes vote down vote up
def gradient_penalty_loss(self, y_true, y_pred, averaged_samples):
        """
        Computes gradient penalty based on prediction and weighted real / fake samples
        """
        gradients = K.gradients(y_pred, averaged_samples)[0]
        # compute the euclidean norm by squaring ...
        gradients_sqr = K.square(gradients)
        #   ... summing over the rows ...
        gradients_sqr_sum = K.sum(gradients_sqr,
                                  axis=np.arange(1, len(gradients_sqr.shape)))
        #   ... and sqrt
        gradient_l2_norm = K.sqrt(gradients_sqr_sum)
        # compute lambda * (1 - ||grad||)^2 still for each single sample
        gradient_penalty = K.square(1 - gradient_l2_norm)
        # return the mean as loss over all the batch samples
        return K.mean(gradient_penalty) 
Example 5
Project: knowledgeflow   Author: 3rduncle   File: lcd.py    MIT License 6 votes vote down vote up
def semantic_matrix(argv):
    assert len(argv) == 2
    q = argv[0]
    a = argv[1]
    q_sqrt = K.sqrt((q ** 2).sum(axis=2, keepdims=True))
    a_sqrt = K.sqrt((a ** 2).sum(axis=2, keepdims=True))
    denominator = K.batch_dot(q_sqrt, K.permute_dimensions(a_sqrt, [0,2,1]))
    return K.batch_dot(q, K.permute_dimensions(a, [0,2,1])) / (denominator + SAFE_EPSILON)

# 注意idx是二维的矩阵
# 如何执行类似batch index的效果折腾了半天
# 参考https://groups.google.com/forum/#!topic/theano-users/7gUdN6E00Dc
# 注意argmax里面是2 - axis
# 注意theano里面a > 0返回的是整数类似[1,1,0]的矩阵,里面的值是整数而不
# 是bool值,不能直接作为索引值
# 因此需要做这样的操作T.set_subtensor(ib[(ib < 0).nonzero()], 0) 
Example 6
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 6 votes vote down vote up
def visualize_model(model, include_gradients=False):
    recurrent_layer = model.get_layer('recurrent_layer')
    output_layer = model.get_layer('output_layer')

    inputs = []
    inputs.extend(model.inputs)

    outputs = []
    outputs.extend(model.outputs)
    outputs.append(recurrent_layer.output)
    outputs.append(recurrent_layer.W_f)  # -- weights of the forget gates (assuming LSTM)

    if include_gradients:
        loss = K.mean(model.output)  # [batch_size, 1] -> scalar
        grads = K.gradients(loss, recurrent_layer.output)
        grads_norm = grads / (K.sqrt(K.mean(K.square(grads))) + 1e-5)
        outputs.append(grads_norm)

    all_function = K.function(inputs, outputs)
    output_function = K.function([output_layer.input], model.outputs)
    return all_function, output_function 
Example 7
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 6 votes vote down vote up
def __init__(self,batch_size):
    self.batch_size = batch_size
    
    def loss_DSSIS_tf11(self, y_true, y_pred):
        """Need tf0.11rc to work"""
        y_true = tf.reshape(y_true, [self.batch_size] + get_shape(y_pred)[1:])
        y_pred = tf.reshape(y_pred, [self.batch_size] + get_shape(y_pred)[1:])
        y_true = tf.transpose(y_true, [0, 2, 3, 1])
        y_pred = tf.transpose(y_pred, [0, 2, 3, 1])
        patches_true = tf.extract_image_patches(y_true, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")
        patches_pred = tf.extract_image_patches(y_pred, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")

        u_true = K.mean(patches_true, axis=3)
        u_pred = K.mean(patches_pred, axis=3)
        var_true = K.var(patches_true, axis=3)
        var_pred = K.var(patches_pred, axis=3)
        std_true = K.sqrt(var_true)
        std_pred = K.sqrt(var_pred)
        c1 = 0.01 ** 2
        c2 = 0.03 ** 2
        ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
        denom = (u_true ** 2 + u_pred ** 2 + c1) * (var_pred + var_true + c2)
        ssim /= denom
        ssim = tf.select(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
        return K.mean(((1.0 - ssim) / 2)) 
Example 8
Project: cdt-ccm-aae   Author: danielegrattarola   File: layers.py    MIT License 6 votes vote down vote up
def call(self, inputs):
        zero = K.constant(0.)

        # Spherical clip
        spherical_clip = self.radius * K.l2_normalize(inputs, -1)
        # Hyperbolic clip
        free_components = inputs[..., :-1]
        bound_component = K.sqrt(K.sum(free_components ** 2, -1)[..., None] + (self.radius ** 2))
        hyperbolic_clip = K.concatenate((free_components, bound_component), -1)

        lt_cond = K.less(self.radius, zero)
        lt_check = K.switch(lt_cond, hyperbolic_clip, inputs)

        gt_cond = K.greater(self.radius, zero)
        output = K.switch(gt_cond, spherical_clip, lt_check)

        return output 
Example 9
Project: CDAE4InfoExtraction   Author: grassknoted   File: capsulelayers.py    MIT License 6 votes vote down vote up
def call(self, inputs, **kwargs):
        if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.
            assert len(inputs) == 2
            inputs, mask = inputs
        else:  # if no true label, mask by the max length of capsules. Mainly used for prediction
            # compute lengths of capsules
            x = K.sqrt(K.sum(K.square(inputs), -1))
            # generate the mask which is a one-hot code.
            # mask.shape=[None, n_classes]=[None, num_capsule]
            mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])

        # inputs.shape=[None, num_capsule, dim_capsule]
        # mask.shape=[None, num_capsule]
        # masked.shape=[None, num_capsule * dim_capsule]
        masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))
        return masked 
Example 10
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1x(y_true, y_pred):
    lx = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.1 * lx) 
Example 11
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1y(y_true, y_pred):
    ly = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.1 * ly)
    #return (50 * ly) 
Example 12
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1z(y_true, y_pred):
    lz = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.1 * lz) 
Example 13
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1rw(y_true, y_pred):
    lw = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    #return (50 * lw)
    return (0.1 * lw) 
Example 14
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1rx(y_true, y_pred):
    lp = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.05 * lp) 
Example 15
Project: mtrl-auto-uav   Author: brunapearson   File: mtrl_network.py    MIT License 5 votes vote down vote up
def euc_loss1rz(y_true, y_pred):
    lq = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    #return (50 * lq)
    return (0.1 * lq) 
Example 16
Project: VisualNN   Author: angelhunt   File: cifar10_cnn_capsule.py    GNU General Public License v3.0 5 votes vote down vote up
def squash(x, axis=-1):
    s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()
    scale = K.sqrt(s_squared_norm) / (0.5 + s_squared_norm)
    return scale * x


# define our own softmax function instead of K.softmax
# because K.softmax can not specify axis. 
Example 17
Project: Kaggle-Statoil-Challenge   Author: adodd202   File: capsulelayers.py    MIT License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1)) 
Example 18
Project: Kaggle-Statoil-Challenge   Author: adodd202   File: capsulelayers.py    MIT License 5 votes vote down vote up
def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors 
Example 19
Project: MODS_ConvNet   Author: santiagolopezg   File: filter_visualize.py    MIT License 5 votes vote down vote up
def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 
Example 20
Project: MODS_ConvNet   Author: santiagolopezg   File: filter_visualize.py    MIT License 5 votes vote down vote up
def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 
Example 21
Project: CapsAttnNet   Author: rstager   File: canlayer.py    MIT License 5 votes vote down vote up
def squash_scale(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale 
Example 22
Project: deep-models   Author: LaurentMazare   File: lstm_ln.py    Apache License 2.0 5 votes vote down vote up
def norm(self, xs, norm_id):
    mu = K.mean(xs, axis=-1, keepdims=True)
    sigma = K.sqrt(K.var(xs, axis=-1, keepdims=True) + 1e-3)
    xs = self.gs[norm_id] * (xs - mu) / (sigma + 1e-3) + self.bs[norm_id]
    return xs 
Example 23
Project: CapsNet   Author: l11x0m7   File: capsule.py    MIT License 5 votes vote down vote up
def squash(s, axis=-1):
    """
    Squash function. This could be viewed as one kind of activations.
    """
    squared_s = K.sum(K.square(s), axis=axis, keepdims=True)
    scale = squared_s / (1 + squared_s) / K.sqrt(squared_s + K.epsilon())
    return scale * s 
Example 24
Project: CapsNet   Author: l11x0m7   File: capsule.py    MIT License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), axis=-1)) 
Example 25
Project: CapsNet   Author: l11x0m7   File: capsule.py    MIT License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        # inputs -> (X, y), then output the mask of y
        # inputs -> X, then output the mask of prediction
        if type(inputs) is list or tuple:
            inputs, mask = inputs
        else:
            pred = K.sqrt(K.sum(K.square(inputs), axis=-1) + K.epsilon())
            mask = K.one_hot(indices=K.argmax(pred, 1), num_classes=pred.get_shape().as_list()[1])
        return K.batch_flatten(inputs * K.expand_dims(mask, axis=-1)) 
Example 26
Project: 3DGCN   Author: blackmints   File: loss.py    MIT License 5 votes vote down vote up
def std_rmse(std=1):
    def rmse(y_true, y_pred):
        return K.sqrt(K.mean(K.square((y_pred - y_true)))) * std

    return rmse 
Example 27
Project: isl-gaze-demo   Author: djpetti   File: metrics.py    MIT License 5 votes vote down vote up
def distance_metric(y_true, y_pred):
  """ Calculates the euclidean distance between the two labels and the
  predictions.
  Args:
    y_true: The true labels.
    y_pred: The predictions.
  Returns:
    The element-wise euclidean distance between the labels and the predictions.
  """
  diff = y_true - y_pred
  sqr = K.square(diff)
  total = K.sum(sqr, axis=1)
  return K.sqrt(total) 
Example 28
Project: isl-gaze-demo   Author: djpetti   File: train_eyes.py    MIT License 5 votes vote down vote up
def distance_metric(y_true, y_pred):
  """ Calculates the euclidean distance between the two labels and the
  predictions.
  Args:
    y_true: The true labels.
    y_pred: The predictions.
  Returns:
    The element-wise euclidean distance between the labels and the predictions.
  """
  diff = y_true - y_pred
  sqr = K.square(diff)
  total = K.sum(sqr, axis=1)
  return K.sqrt(total) 
Example 29
Project: fancy-cnn   Author: textclf   File: embeddings.py    MIT License 5 votes vote down vote up
def __call__(self, p):
        if self.skip:
            return self.s * (p / K.clip(K.sqrt(K.sum(K.square(p), axis=-1, keepdims=True)), 0.5, 100))
        return self.s * (p / K.sqrt(K.sum(K.square(p), axis=-1, keepdims=True))) 
Example 30
Project: fancy-cnn   Author: textclf   File: embeddings.py    MIT License 5 votes vote down vote up
def __call__(self, p):
        if self.skip:
            return self.s * (p / K.clip(K.sqrt(K.sum(K.square(p), axis=-1, keepdims=True)), 0.5, 100))
        return self.s * (p / K.sqrt(K.sum(K.square(p), axis=-1, keepdims=True))) 
Example 31
Project: BlurbGenreCollection-HMC   Author: uhh-lt   File: capsulelayers.py    Apache License 2.0 5 votes vote down vote up
def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1)) 
Example 32
Project: BlurbGenreCollection-HMC   Author: uhh-lt   File: capsulelayers.py    Apache License 2.0 5 votes vote down vote up
def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors 
Example 33
Project: CNNLocalizationTF   Author: hulop   File: posenet_loss.py    MIT License 5 votes vote down vote up
def euc_loss1x(y_true, y_pred):
    lx = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.3 * lx) 
Example 34
Project: CNNLocalizationTF   Author: hulop   File: posenet_loss.py    MIT License 5 votes vote down vote up
def euc_loss1q(y_true, y_pred):
    lq = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (150 * lq) 
Example 35
Project: CNNLocalizationTF   Author: hulop   File: posenet_loss.py    MIT License 5 votes vote down vote up
def euc_loss2x(y_true, y_pred):
    lx = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (0.3 * lx) 
Example 36
Project: CNNLocalizationTF   Author: hulop   File: posenet_loss.py    MIT License 5 votes vote down vote up
def euc_loss2q(y_true, y_pred):
    lq = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (150 * lq) 
Example 37
Project: CNNLocalizationTF   Author: hulop   File: posenet_loss.py    MIT License 5 votes vote down vote up
def euc_loss3x(y_true, y_pred):
    lx = K.sqrt(K.sum(K.square(y_true[:,:] - y_pred[:,:]), axis=1, keepdims=True))
    return (1 * lx) 
Example 38
Project: CNNLocalizationTF   Author: hulop   File: posenet_lstm_loss.py    MIT License 5 votes vote down vote up
def euc_loss3x(y_true, y_pred):
    lx = K.sqrt(K.sum(K.sum(K.square(y_true[:,:,:] - y_pred[:,:,:]), axis=2, keepdims=True), axis=1, keepdims=True))
    return (1 * lx) 
Example 39
Project: CNNLocalizationTF   Author: hulop   File: posenet_lstm_loss.py    MIT License 5 votes vote down vote up
def euc_loss3q(y_true, y_pred):
    lq = K.sqrt(K.sum(K.sum(K.square(y_true[:,:,:] - y_pred[:,:,:]), axis=2, keepdims=True), axis=1, keepdims=True))
    return (500 * lq) 
Example 40
Project: keras-lamb   Author: CyberZHG   File: optimizer.py    MIT License 5 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1
        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * t))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            mhat_t = m_t / (1. - K.pow(self.beta_1, t))
            vhat_t = v_t / (1. - K.pow(self.beta_2, t))

            u_t = mhat_t / K.sqrt(vhat_t + self.epsilon) + self.weight_decay * p
            trust_ratio = K.sqrt(K.sum(K.square(p)) / K.sum(K.square(u_t)))
            trust_ratio = K.minimum(K.maximum(trust_ratio, self.lower_bound), self.upper_bound)

            lr_p = trust_ratio * lr
            new_p = p - lr_p * u_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 41
Project: cbc_networks   Author: saralajew   File: constraints.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __call__(self, w):
        w /= K.sqrt(K.maximum(K.sum(K.square(w),
                                    self.axis,
                                    keepdims=True),
                              K.epsilon()))
        return w 
Example 42
Project: cbc_networks   Author: saralajew   File: detection_probability_functions.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        def sqrt(x):
            return K.sqrt(K.maximum(x, K.epsilon()))

        # Both components and input given
        if isinstance(inputs, list) and len(inputs) > 1:
            signals, kernel = inputs
        else:
            signals = inputs
            kernel = self.components.astype(K.floatx())

        # move component_number to channel dimension
        kernel = K.permute_dimensions(kernel, (1, 2, 3, 0))
        # normalize kernel
        normed_kernel = kernel / sqrt(K.sum(K.square(kernel),
                                            (0, 1, 2),
                                            keepdims=True))

        # get norm of signals
        signals_norm = sqrt(K.conv2d(K.square(signals),
                                     np.ones(K.int_shape(kernel)[:3] + (1,),
                                             dtype=K.floatx()),
                                     strides=self.strides,
                                     padding=self.padding,
                                     data_format='channels_last',
                                     dilation_rate=self.dilation_rate))

        diss = K.conv2d(signals,
                        normed_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format='channels_last',
                        dilation_rate=self.dilation_rate) / signals_norm

        if self.n_replicas != 1:
            shape = K.int_shape(diss)
            diss = K.reshape(diss, (-1, shape[1], shape[2],
                                    shape[3] // self.n_replicas,
                                    self.n_replicas))

        return self.activation(diss) 
Example 43
Project: Neural_Temporality_Adaptation   Author: xiaoleihuang   File: TEC_gru.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance_loss(params, params_prev):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    :param params: the current model parameters
    :param params_prev: previous model parameters
    :return: float
    """
    return K.sqrt(K.sum(K.square(params - params_prev), axis=-1)) 
Example 44
Project: Neural_Temporality_Adaptation   Author: xiaoleihuang   File: TEC_gru_dia.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance_loss(params, params_prev):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    :param params: the current model parameters
    :param params_prev: previous model parameters
    :return: float
    """
    return K.sqrt(K.sum(K.square(params - params_prev), axis=-1)) 
Example 45
Project: Neural_Temporality_Adaptation   Author: xiaoleihuang   File: TEC_cnn_dia.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance_loss(params, params_prev):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    :param params: the current model parameters
    :param params_prev: previous model parameters
    :return: float
    """
    return K.sqrt(K.sum(K.square(params - params_prev), axis=-1)) 
Example 46
Project: Neural_Temporality_Adaptation   Author: xiaoleihuang   File: TEC_cnn_sig.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance_loss(params, params_prev):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    :param params: the current model parameters
    :param params_prev: previous model parameters
    :return: float
    """
    return K.sqrt(K.sum(K.square(params - params_prev), axis=-1)) 
Example 47
Project: Neural_Temporality_Adaptation   Author: xiaoleihuang   File: TEC_cnn.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance_loss(params, params_prev):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    :param params: the current model parameters
    :param params_prev: previous model parameters
    :return: float
    """
    return K.sqrt(K.sum(K.square(params - params_prev), axis=-1)) 
Example 48
Project: Keras_MedicalImgAI   Author: taoyilee   File: grad_cam.py    MIT License 5 votes vote down vote up
def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 
Example 49
Project: 360_aware_saliency   Author: MikhailStartsev   File: models.py    GNU General Public License v3.0 5 votes vote down vote up
def correlation_coefficient(y_true, y_pred):
    max_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)), 
                                                                   shape_r_out, axis=-1)), shape_c_out, axis=-1)
    y_pred /= max_y_pred

    sum_y_true = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_true, axis=2), axis=2)), 
                                                                   shape_r_out, axis=-1)), shape_c_out, axis=-1)
    sum_y_pred = K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(K.sum(K.sum(y_pred, axis=2), axis=2)), 
                                                                   shape_r_out, axis=-1)), shape_c_out, axis=-1)

    y_true /= (sum_y_true + K.epsilon())
    y_pred /= (sum_y_pred + K.epsilon())

    N = shape_r_out * shape_c_out
    sum_prod = K.sum(K.sum(y_true * y_pred, axis=2), axis=2)
    sum_x = K.sum(K.sum(y_true, axis=2), axis=2)
    sum_y = K.sum(K.sum(y_pred, axis=2), axis=2)
    sum_x_square = K.sum(K.sum(K.square(y_true), axis=2), axis=2)
    sum_y_square = K.sum(K.sum(K.square(y_pred), axis=2), axis=2)

    num = sum_prod - ((sum_x * sum_y) / N)
    den = K.sqrt((sum_x_square - K.square(sum_x) / N) * (sum_y_square - K.square(sum_y) / N))

    return -2 * num / den


# Normalized Scanpath Saliency Loss 
Example 50
Project: DisplaceNet   Author: GKalliatakis   File: generic_utils.py    MIT License 5 votes vote down vote up
def euclidean_distance_loss(y_true, y_pred):
    """
    Euclidean distance loss
    https://en.wikipedia.org/wiki/Euclidean_distance
    http://www.riptutorial.com/keras/example/32022/euclidean-distance-loss
    :param y_true: TensorFlow/Theano tensor
    :param y_pred: TensorFlow/Theano tensor of the same shape as y_true
    :return: float
    """
    return K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1)) 
Example 51
Project: DisplaceNet   Author: GKalliatakis   File: generic_utils.py    MIT License 5 votes vote down vote up
def rmse(y_true, y_pred):
    """
    Root mean squared error
    https://en.wikipedia.org/wiki/Euclidean_distance
    http://www.riptutorial.com/keras/example/32022/euclidean-distance-loss
    :param y_true: TensorFlow/Theano tensor
    :param y_pred: TensorFlow/Theano tensor of the same shape as y_true
    :return: float
    """
    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) 
Example 52
Project: DisplaceNet   Author: GKalliatakis   File: generic_utils.py    MIT License 5 votes vote down vote up
def weighted_euclidean_distance(self, y_true, y_pred):
        nb_cl = len(self.weights)
        final_mask = K.zeros_like(y_pred[..., 0])
        y_pred_max = K.max(y_pred, axis=-1)
        y_pred_max = K.expand_dims(y_pred_max, axis=-1)
        y_pred_max_mat = K.equal(y_pred, y_pred_max)
        for c_p, c_t in itertools.product(range(nb_cl), range(nb_cl)):
            w = K.cast(self.weights[c_t, c_p], K.floatx())
            y_p = K.cast(y_pred_max_mat[..., c_p], K.floatx())
            y_t = K.cast(y_pred_max_mat[..., c_t], K.floatx())
            final_mask += w * y_p * y_t
        return K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1)) * final_mask 
Example 53
Project: kutils   Author: subpic   File: tensor_ops.py    MIT License 5 votes vote down vote up
def earth_mover_loss(y_true, y_pred):
    """
    Earth Mover's Distance loss.

    Reproduced from https://github.com/titu1994/neural-image-assessment/blob/master/train_inception_resnet.py
    """
    cdf_ytrue = K.cumsum(y_true, axis=-1)
    cdf_ypred = K.cumsum(y_pred, axis=-1)
    samplewise_emd = K.sqrt(K.mean(K.square(K.abs(cdf_ytrue - cdf_ypred)), axis=-1))
    return K.mean(samplewise_emd) 
Example 54
Project: CapsNet-Fashion-MNIST   Author: subarnop   File: capsulelayers.py    GNU General Public License v3.0 5 votes vote down vote up
def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1)) 
Example 55
Project: CapsNet-Fashion-MNIST   Author: subarnop   File: capsulelayers.py    GNU General Public License v3.0 5 votes vote down vote up
def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors 
Example 56
Project: keras-adamw   Author: OverLordGoldDragon   File: utils.py    MIT License 5 votes vote down vote up
def _apply_weight_decays(cls, var, var_t):
    wd = cls.weight_decays[var.name]
    wd_normalized = wd * K.cast(
            K.sqrt(cls.batch_size / cls.total_iterations_wd), 'float32')
    var_t = var_t - cls.eta_t * wd_normalized * var

    if cls.init_verbose and not cls._init_notified:
        print('{} weight decay set for {}'.format(
                K_eval(wd_normalized), var.name))
    return var_t 
Example 57
Project: knowledgeflow   Author: 3rduncle   File: lcd.py    MIT License 5 votes vote down vote up
def compute_similar(q, a):
    q_sqrt = K.sqrt((q ** 2).sum(axis=1))
    a_sqrt = K.sqrt((a ** 2).sum(axis=1))
    denominator = q_sqrt * a_sqrt
    output = (q * a).sum(axis=1) / (denominator + SAFE_EPSILON)
    return K.expand_dims(output, 1) 
Example 58
Project: DeepLearn   Author: GauravBh1010tt   File: DeepLearn_cornet.py    MIT License 5 votes vote down vote up
def cor(self,y1, y2, lamda):
        y1_mean = K.mean(y1, axis=0)
        y1_centered = y1 - y1_mean
        y2_mean = K.mean(y2, axis=0)
        y2_centered = y2 - y2_mean
        corr_nr = K.sum(y1_centered * y2_centered, axis=0)
        corr_dr1 = K.sqrt(T.sum(y1_centered * y1_centered, axis=0) + 1e-8)
        corr_dr2 = K.sqrt(T.sum(y2_centered * y2_centered, axis=0) + 1e-8)
        corr_dr = corr_dr1 * corr_dr2
        corr = corr_nr / corr_dr
        return K.sum(corr) * lamda 
Example 59
Project: DeepLearn   Author: GauravBh1010tt   File: DeepLearn_cornet.py    MIT License 5 votes vote down vote up
def sum_corr(model):
    view1 = np.load("test_v1.npy")
    view2 = np.load("test_v2.npy")
    x = project(model,[view1,np.zeros_like(view1)])
    y = project(model,[np.zeros_like(view2),view2])
    print "test correlation"
    corr = 0
    for i in range(0,len(x[0])):
		x1 = x[:,i] - (np.ones(len(x))*(sum(x[:,i])/len(x)))
		x2 = y[:,i] - (np.ones(len(y))*(sum(y[:,i])/len(y)))
		nr = sum(x1 * x2)/(math.sqrt(sum(x1*x1))*math.sqrt(sum(x2*x2)))
		corr+=nr
    print corr 
Example 60
Project: DeepLearn   Author: GauravBh1010tt   File: model_abcnn.py    MIT License 5 votes vote down vote up
def compute_euclidean_match_score(l_r):
    l, r = l_r
    denominator = 1. + K.sqrt(
        -2 * K.batch_dot(l, r, axes=[2, 2]) +
        K.expand_dims(K.sum(K.square(l), axis=2), 2) +
        K.expand_dims(K.sum(K.square(r), axis=2), 1)
    )
    denominator = K.maximum(denominator, K.epsilon())
    return 1. / denominator 
Example 61
Project: DeepLearn   Author: GauravBh1010tt   File: XRMB_CNN_17.06.v2.py    MIT License 5 votes vote down vote up
def cor(self,y1, y2, lamda):
        y1_mean = K.mean(y1, axis=0)
        y1_centered = y1 - y1_mean
        y2_mean = K.mean(y2, axis=0)
        y2_centered = y2 - y2_mean
        corr_nr = K.sum(y1_centered * y2_centered, axis=0)
        corr_dr1 = K.sqrt(K.sum(y1_centered * y1_centered, axis=0) + 1e-8)
        corr_dr2 = K.sqrt(K.sum(y2_centered * y2_centered, axis=0) + 1e-8)
        corr_dr = corr_dr1 * corr_dr2
        corr = corr_nr / corr_dr
        return K.sum(corr) * lamda 
Example 62
Project: DeepLearn   Author: GauravBh1010tt   File: XRMB_CNN_17.06.v2.py    MIT License 5 votes vote down vote up
def sum_corr(model):
    view1 = np.load("MFCC_Test.npy")
    view2 = np.load("XRMB_Test.npy")
    x = project(model,[view1,np.zeros_like(view2)])
    y = project(model,[np.zeros_like(view1),view2])
    print ("test correlation")
    corr = 0
    for i in range(0,len(x[0])):
        x1 = x[:,i] - (np.ones(len(x))*(sum(x[:,i])/len(x)))
        x2 = y[:,i] - (np.ones(len(y))*(sum(y[:,i])/len(y)))
        nr = sum(x1 * x2)/(math.sqrt(sum(x1*x1))*math.sqrt(sum(x2*x2)))
        corr+=nr
    print (corr) 
Example 63
Project: DeepLearn   Author: GauravBh1010tt   File: CorrMCNN_Arch2.py    MIT License 5 votes vote down vote up
def sum_corr(model):
    view1 = np.load("test_v1.npy")
    view2 = np.load("test_v2.npy")
    x = project(model,[view1,np.zeros_like(view1)])
    y = project(model,[np.zeros_like(view2),view2])
    print ("test correlation")
    corr = 0
    for i in range(0,len(x[0])):
        x1 = x[:,i] - (np.ones(len(x))*(sum(x[:,i])/len(x)))
        x2 = y[:,i] - (np.ones(len(y))*(sum(y[:,i])/len(y)))
        nr = sum(x1 * x2)/(math.sqrt(sum(x1*x1))*math.sqrt(sum(x2*x2)))
        corr+=nr
    print (corr) 
Example 64
Project: gccaps   Author: tqbl   File: capsules.py    MIT License 5 votes vote down vote up
def squash(x, axis=-1):
    """Apply a squashing nonlinearity as described in [1]_.

    Args:
        x (tensor): Input tensor to transform.
        axis (int): Axis along which squashing is applied.

    Returns:
        A Keras tensor of the resulting output.
    """
    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) \
        / K.sqrt(s_squared_norm + K.epsilon())
    return scale * x 
Example 65
Project: gccaps   Author: tqbl   File: capsules.py    MIT License 5 votes vote down vote up
def length(x):
    """Compute the Euclidean lengths of capsules.

    Args:
        x (tensor): Tensor of capsules.

    Returns:
        A Keras tensor of lengths.
    """
    return K.sqrt(K.sum(K.square(x), -1)) 
Example 66
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 5 votes vote down vote up
def euclidean_distance(inputs):
    assert len(inputs) == 2, \
        'Euclidean distance needs 2 inputs, %d given' % len(inputs)
    u, v = inputs
    return K.sqrt((K.square(u - v)).sum(axis=1, keepdims=True)) 
Example 67
Project: face_classification   Author: oarriaga   File: grad_cam.py    MIT License 5 votes vote down vote up
def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 
Example 68
Project: deepkeyphraseextraction   Author: basaldella   File: AnswerRNN2.py    Apache License 2.0 5 votes vote down vote up
def cos_distance(y_true, y_pred):
    import keras.backend as K
    def l2_normalize(x, axis):
        norm = K.sqrt(K.sum(K.square(x), axis=axis, keepdims=True))
        return K.sign(x) * K.maximum(K.abs(x), K.epsilon()) / K.maximum(norm, K.epsilon())

    y_true = K.l2_normalize(y_true, axis=-1)
    y_pred = K.l2_normalize(y_pred, axis=-1)
    return K.mean(1 - K.sum((y_true * y_pred), axis=-1))

# End loss 
Example 69
Project: FaceLandmarks   Author: JACKYLUO1991   File: loss.py    Apache License 2.0 5 votes vote down vote up
def normalized_mean_error(y_true, y_pred):
    '''
    normalised mean error
    '''
    y_pred = K.reshape(y_pred, (-1, N_LANDMARK, 2))
    y_true = K.reshape(y_true, (-1, N_LANDMARK, 2))
    # Distance between pupils
    interocular_distance = K.sqrt(
        K.sum((y_true[:, 38, :] - y_true[:, 92, :]) ** 2, axis=-1))
    return K.mean(K.sum(K.sqrt(K.sum((y_pred - y_true) ** 2, axis=-1)), axis=-1)) / \
        K.mean((interocular_distance * N_LANDMARK))


# def wing_loss(y_true, y_pred, w=10.0, epsilon=2.0):
#     """
#     Reference: wing loss for robust facial landmark localisation
#     with convolutional neural networks
#     """
#     x = y_true - y_pred
#     c = w * (1.0 - math.log(1.0 + w/epsilon))
#     absolute_x = K.abs(x)
#     losses = tf.where(
#         K.greater(w, absolute_x),
#         w * K.log(1.0 + absolute_x/epsilon),
#         absolute_x - c
#     )
#     loss = K.mean(K.sum(losses, axis=-1), axis=0)

#     return loss 
Example 70
Project: Emotion   Author: petercunha   File: grad_cam.py    MIT License 5 votes vote down vote up
def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 
Example 71
Project: CDAE4InfoExtraction   Author: grassknoted   File: capsulelayers.py    MIT License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1)) 
Example 72
Project: CDAE4InfoExtraction   Author: grassknoted   File: capsulelayers.py    MIT License 5 votes vote down vote up
def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors 
Example 73
Project: BERT   Author: yyht   File: layers.py    Apache License 2.0 5 votes vote down vote up
def call(self, x, **kwargs):
        u = K.mean(x, axis=-1, keepdims=True)
        s = K.mean(K.square(x - u), axis=-1, keepdims=True)
        z = (x - u) / K.sqrt(s + self.eps)
        return self.gamma * z + self.beta 
Example 74
Project: bert_lamb_pretrain   Author: goldenbili   File: keras_lamb.py    Apache License 2.0 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            m_t_hat = m_t / (1. - K.pow(self.beta_1, t))
            v_t_hat = v_t / (1. - K.pow(self.beta_2, t))

            p_dash = m_t_hat / (K.sqrt(v_t_hat + self.epsilon))

            if self.weight_decay > 0.:
                wd = self.weight_decay * p
                p_dash = p_dash + wd

            r1 = K.sqrt(K.sum(K.square(p)))
            r2 = K.sqrt(K.sum(K.square(p_dash)))

            r = tf.where(tf.greater(r1, 0.),
                         tf.where(tf.greater(r2, 0.),
                                  r1 / r2,
                                  1.0),
                         1.0)
            # r = r1 / r2
            eta = r * lr

            p_t = p - eta * p_dash

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 75
Project: bert_lamb_pretrain   Author: goldenbili   File: keras_lamb.py    Apache License 2.0 4 votes vote down vote up
def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        """See base class."""
        assignments = []
        for (grad, param) in grads_and_vars:
            if grad is None or param is None:
                continue

            param_name = self._get_variable_name(param.name)

            m = tf.get_variable(
                name=param_name + "/adam_m",
                shape=param.shape.as_list(),
                dtype=tf.float32,
                trainable=False,
                initializer=tf.zeros_initializer())
            v = tf.get_variable(
                name=param_name + "/adam_v",
                shape=param.shape.as_list(),
                dtype=tf.float32,
                trainable=False,
                initializer=tf.zeros_initializer())

            # Standard Adam update.
            next_m = (
                    tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad))
            next_v = (
                    tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2,
                                                              tf.square(grad)))

            update = next_m / (tf.sqrt(next_v) + self.epsilon)

            # Just adding the square of the weights to the loss function is *not*
            # the correct way of using L2 regularization/weight decay with Adam,
            # since that will interact with the m and v parameters in strange ways.
            #
            # Instead we want ot decay the weights in a manner that doesn't interact
            # with the m/v parameters. This is equivalent to adding the square
            # of the weights to the loss with plain (non-momentum) SGD.
            if self._do_use_weight_decay(param_name):
                update += self.weight_decay_rate * param

            update_with_lr = self.learning_rate * update

            next_param = param - update_with_lr

            assignments.extend(
                [param.assign(next_param),
                 m.assign(next_m),
                 v.assign(next_v)])
        return tf.group(*assignments, name=name) 
Example 76
Project: keras_radam   Author: forin-xyz   File: radam.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay:
            lr = lr * (1. / (1. + self.decay * K.cast(
                self.iterations, K.dtype(self.decay)
            )))

        t = K.cast(self.iterations, K.floatx()) + 1.
        beta_1 = self.beta_1
        beta_2 = self.beta_2
        beta_1_t = K.pow(beta_1, t)
        beta_2_t = K.pow(beta_2, t)
        rho_inf = 2. / (1. - beta_2) - 1.
        rho_t = rho_inf - 2. * t * beta_2_t / (1. - beta_2_t)
        r_t = K.sqrt(
            K.relu(rho_t - 4.) * (rho_t - 2.) * rho_inf / (
                (rho_inf - 4.) * (rho_inf - 2.) * rho_t )
        )
        flag = K.cast(rho_t > 4., K.floatx())

        ms = [K.zeros(K.int_shape(p)) for p in params]
        vs = [K.zeros(K.int_shape(p)) for p in params]

        self.weights = [self.iterations] + ms + vs
        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = beta_1 * m + (1. - beta_1) * g
            v_t = beta_2 * v + (1. - beta_2) * K.square(g)

            m_hat_t = m_t / (1. - beta_1_t)
            v_hat_t = K.sqrt(v_t / (1. - beta_2_t))
            new_p = p - lr * (r_t / (v_hat_t + self.epsilon) + flag - 1.)* m_hat_t

            if getattr(p, "constraint", None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
        return self.updates 
Example 77
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]

        if self.amsgrad:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        else:
            vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = lr_t * self.eta_t  # for external tracking
        lr_t_premult = lr_t

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            # Learning rate multipliers
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t_premult, p)

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            if self.amsgrad:
                vhat_t = K.maximum(vhat, v_t)
                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, vhat_t))
            else:
                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 78
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        t = K.cast(self.iterations, K.floatx()) + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = self.lr * self.eta_t  # for external tracking

        for p, g, m, v in zip(params, grads, ms, vs):
            # Learning rate multipliers
            lr_t = self.lr
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t, p)

            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + (
                momentum_cache_t_1 * m_t_prime)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            p_t = p - self.eta_t * lr_t * m_t_bar / (
                    K.sqrt(v_t_prime) + self.epsilon)

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 79
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 80
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates