Python keras.backend.floatx() Examples

The following are code examples for showing how to use keras.backend.floatx(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: cdc   Author: ckbjimmy   File: Attention.py    MIT License 6 votes vote down vote up
def call(self, x, mask=None):
        eij = dot_product(x, self.W)

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1) 
Example 2
Project: GlyphsScripts   Author: simoncozens   File: Autokern.py    MIT License 6 votes vote down vote up
def w_categorical_crossentropy(self, y_true, y_pred):
    nb_cl = len(self.weights)
    final_mask = K.zeros_like(y_pred[..., 0])
    y_pred_max = K.max(y_pred, axis=-1)
    y_pred_max = K.expand_dims(y_pred_max, axis=-1)
    y_pred_max_mat = K.equal(y_pred, y_pred_max)
    for c_p, c_t in itertools.product(range(nb_cl), range(nb_cl)):
        w = K.cast(self.weights[c_t, c_p], K.floatx())
        y_p = K.cast(y_pred_max_mat[..., c_p], K.floatx())
        y_t = K.cast(y_true[..., c_t], K.floatx())
        final_mask += w * y_p * y_t
    return K.categorical_crossentropy(y_pred, y_true) * final_mask 
Example 3
Project: dialectal_arabic_segmenter   Author: qcri   File: ChainCRF.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def add_boundary_energy(x, b_start=None, b_end=None, mask=None):
    '''Given the observations x, it adds the start boundary energy b_start (resp.
    end boundary energy b_end on the start (resp. end) elements and multiplies
    the mask.'''
    if mask is None:
        if b_start is not None:
            x = K.concatenate([x[:, :1, :] + b_start, x[:, 1:, :]], axis=1)
        if b_end is not None:
            x = K.concatenate([x[:, :-1, :], x[:, -1:, :] + b_end], axis=1)
    else:
        mask = K.cast(mask, K.floatx())
        mask = K.expand_dims(mask, 2)
        x *= mask
        if b_start is not None:
            mask_r = K.concatenate([K.zeros_like(mask[:, :1]), mask[:, :-1]], axis=1)
            start_mask = K.cast(K.greater(mask, mask_r), K.floatx())
            x = x + start_mask * b_start
        if b_end is not None:
            mask_l = K.concatenate([mask[:, 1:], K.zeros_like(mask[:, -1:])], axis=1)
            end_mask = K.cast(K.greater(mask, mask_l), K.floatx())
            x = x + end_mask * b_end
    return x 
Example 4
Project: dialectal_arabic_segmenter   Author: qcri   File: ChainCRF.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def _forward(x, reduce_step, initial_states, U, mask=None):
    '''Forward recurrence of the linear chain crf.'''

    def _forward_step(energy_matrix_t, states):
        alpha_tm1 = states[-1]
        new_states = reduce_step(K.expand_dims(alpha_tm1, 2) + energy_matrix_t)
        return new_states[0], new_states

    U_shared = K.expand_dims(K.expand_dims(U, 0), 0)

    if mask is not None:
        mask = K.cast(mask, K.floatx())
        mask_U = K.expand_dims(K.expand_dims(mask[:, :-1] * mask[:, 1:], 2), 3)
        U_shared = U_shared * mask_U

    inputs = K.expand_dims(x[:, 1:, :], 2) + U_shared
    inputs = K.concatenate([inputs, K.zeros_like(inputs[:, -1:, :, :])], axis=1)

    last, values, _ = K.rnn(_forward_step, inputs, initial_states)
    return last, values 
Example 5
Project: fcd_torch   Author: insilicomedicine   File: keras2pytorch.py    MIT License 6 votes vote down vote up
def build_masked_loss(loss_function, mask_value):
    """Builds a loss function that masks based on targets

    Args:
        loss_function: The loss function to mask
        mask_value: The value to mask in the targets

    Returns:
        function: a loss function that acts like loss_function
        with masked inputs.
    """

    def masked_loss_function(y_true, y_pred):
        mask = K.cast(K.not_equal(y_true, mask_value), K.floatx())
        return loss_function(y_true * mask, y_pred * mask)

    return masked_loss_function 
Example 6
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 6 votes vote down vote up
def call(self, inputs, training=None):

        def dropped_inputs():
            outputs = inputs
            if self.data_format == 'channels_first':
                outputs = K.permute_dimensions(outputs, [0, 2, 1])
            shape = K.shape(outputs)
            if self.sync_channels:
                mask = self._compute_drop_mask([shape[0], shape[1], 1])
            else:
                mask = self._compute_drop_mask(shape)
            outputs = outputs * mask *\
                (K.cast(K.prod(shape), dtype=K.floatx()) / K.sum(mask))
            if self.data_format == 'channels_first':
                outputs = K.permute_dimensions(outputs, [0, 2, 1])
            return outputs

        return K.in_train_phase(dropped_inputs, inputs, training=training) 
Example 7
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 6 votes vote down vote up
def call(self, inputs, training=None):

        def dropped_inputs():
            outputs = inputs
            if self.data_format == 'channels_first':
                outputs = K.permute_dimensions(outputs, [0, 2, 3, 1])
            shape = K.shape(outputs)
            if self.sync_channels:
                mask = self._compute_drop_mask([shape[0], shape[1], shape[2], 1])
            else:
                mask = self._compute_drop_mask(shape)
            outputs = outputs * mask *\
                (K.cast(K.prod(shape), dtype=K.floatx()) / K.sum(mask))
            if self.data_format == 'channels_first':
                outputs = K.permute_dimensions(outputs, [0, 3, 1, 2])
            return outputs

        return K.in_train_phase(dropped_inputs, inputs, training=training) 
Example 8
Project: humpback-whale-4th-place   Author: daustingm1   File: ensemble.py    Apache License 2.0 6 votes vote down vote up
def __getitem__(self, index):
        start = self.batch_size * index
        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))
        size = end - start
        assert size > 0
        a = np.zeros((size,) + img_shape, dtype=K.floatx())
        b = np.zeros((size,) + img_shape, dtype=K.floatx())
        c = np.zeros((size, 1), dtype=K.floatx())
        j = start // 2
        for i in range(0, size, 2):
            a[i, :, :, :] = read_for_training(self.match[j][0])
            b[i, :, :, :] = read_for_training(self.match[j][1])
            c[i, 0] = 1  # This is a match
            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
            c[i + 1, 0] = 0  # Different whales
            j += 1
        return [a, b], c 
Example 9
Project: humpback-whale-4th-place   Author: daustingm1   File: siamese_train.py    Apache License 2.0 6 votes vote down vote up
def __getitem__(self, index):
        start = self.batch_size * index
        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))
        size = end - start
        assert size > 0
        a = np.zeros((size,) + img_shape, dtype=K.floatx())
        b = np.zeros((size,) + img_shape, dtype=K.floatx())
        c = np.zeros((size, 1), dtype=K.floatx())
        j = start // 2
        for i in range(0, size, 2):
            a[i, :, :, :] = read_for_training(self.match[j][0])
            b[i, :, :, :] = read_for_training(self.match[j][1])
            c[i, 0] = 1  # This is a match
            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
            c[i + 1, 0] = 0  # Different whales
            j += 1
        return [a, b], c 
Example 10
Project: humpback-whale-4th-place   Author: daustingm1   File: siamese_inference.py    Apache License 2.0 6 votes vote down vote up
def __getitem__(self, index):
        start = self.batch_size * index
        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))
        size = end - start
        assert size > 0
        a = np.zeros((size,) + img_shape, dtype=K.floatx())
        b = np.zeros((size,) + img_shape, dtype=K.floatx())
        c = np.zeros((size, 1), dtype=K.floatx())
        j = start // 2
        for i in range(0, size, 2):
            a[i, :, :, :] = read_for_training(self.match[j][0])
            b[i, :, :, :] = read_for_training(self.match[j][1])
            c[i, 0] = 1  # This is a match
            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
            c[i + 1, 0] = 0  # Different whales
            j += 1
        return [a, b], c 
Example 11
Project: DeepResearch   Author: Hsankesara   File: attention_with_context.py    MIT License 6 votes vote down vote up
def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)
        ait = dot_product(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1) 
Example 12
Project: BERT   Author: yyht   File: model.py    Apache License 2.0 6 votes vote down vote up
def create_transformer(embedding_dim: int = 768, embedding_dropout: float = 0.1,
                       vocab_size: int = 30000, max_len: int = 512,
                       trainable_pos_embedding: bool = True, num_heads: int = 12, num_layers: int = 12,
                       attention_dropout: float = 0.1, use_one_embedding_dropout: bool = False,
                       d_hid: int = 768 * 4, residual_dropout: float = 0.1,
                       use_attn_mask: bool = True) -> keras.Model:
    vocab_size += TextEncoder.SPECIAL_COUNT
    tokens = Input(batch_shape=(None, max_len), name='token_input', dtype='int32')
    segment_ids = Input(batch_shape=(None, max_len), name='segment_input', dtype='int32')
    pos_ids = Input(batch_shape=(None, max_len), name='position_input', dtype='int32')
    attn_mask = Input(batch_shape=(None, 1, max_len, max_len), name='attention_mask_input',
                      dtype=K.floatx()) if use_attn_mask else None
    inputs = [tokens, segment_ids, pos_ids]
    embedding_layer = Embedding(embedding_dim, embedding_dropout, vocab_size, max_len, trainable_pos_embedding,
                                use_one_embedding_dropout)
    x = embedding_layer(inputs)
    for i in range(num_layers):
        x = EncoderLayer(embedding_dim, num_heads, d_hid, residual_dropout,
                         attention_dropout, use_attn_mask, i)(x, attn_mask)
    inputs = inputs + ([attn_mask] if use_attn_mask else [])
    return keras.Model(inputs=inputs, outputs=x, name='Transformer') 
Example 13
Project: group-ksparse-temporal-cnns   Author: srph25   File: ops.py    MIT License 6 votes vote down vote up
def ksparse(x, k, axis, alpha=1, absolute=False):
    if isinstance(axis, int):
        axis = (axis,)
    elif isinstance(axis, list):
        axis = tuple(axis)
    axis_complement = tuple(set(range(K.ndim(x))) - set(axis))
    shape_reduce = K.prod([K.shape(x)[j] for j in axis])
    _k = K.minimum(K.in_train_phase(k, alpha * k), shape_reduce)
    inputs_permute_dimensions = K.permute_dimensions(x, axis_complement + axis)
    inputs_permute_dimensions_reshape = K.reshape(inputs_permute_dimensions, (-1, shape_reduce))
    if absolute is True:
        inputs_permute_dimensions_reshape = K.abs(inputs_permute_dimensions_reshape)
    _, indices = tf.nn.top_k(inputs_permute_dimensions_reshape, _k)
    scatter_indices = K.concatenate([(K.arange(K.shape(inputs_permute_dimensions_reshape)[0])[:, None] * K.ones((1, _k), dtype='int32'))[:, :, None], indices[:, :, None]])
    scatter_updates = K.ones((K.shape(inputs_permute_dimensions_reshape)[0], _k))
    mask_permute_dimensions_reshape = K.cast(tf.scatter_nd(scatter_indices, scatter_updates, K.shape(inputs_permute_dimensions_reshape)), K.floatx())
    mask_permute_dimensions = K.reshape(mask_permute_dimensions_reshape, K.shape(inputs_permute_dimensions))
    mask = K.permute_dimensions(mask_permute_dimensions, tuple(np.argsort(axis_complement + axis)))
    return mask * x 
Example 14
Project: iMIMIC-RCVs   Author: medgift   File: optimizer.py    MIT License 6 votes vote down vote up
def _get_seed_input(self, seed_input):
        """Creates a random `seed_input` if None. Otherwise:
            - Ensures batch_size dim on provided `seed_input`.
            - Shuffle axis according to expected `image_data_format`.
        """
        desired_shape = (1, ) + K.int_shape(self.input_tensor)[1:]
        if seed_input is None:
            return utils.random_array(desired_shape, mean=np.mean(self.input_range),
                                      std=0.05 * (self.input_range[1] - self.input_range[0]))

        # Add batch dim if needed.
        if len(seed_input.shape) != len(desired_shape):
            seed_input = np.expand_dims(seed_input, 0)

        # Only possible if channel idx is out of place.
        if seed_input.shape != desired_shape:
            seed_input = np.moveaxis(seed_input, -1, 1)
        return seed_input.astype(K.floatx()) 
Example 15
Project: keras_bn_library   Author: bnsnapper   File: test_qrnn.py    MIT License 6 votes vote down vote up
def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
	"""Generates an absolute cosine time series with the amplitude
	exponentially decreasing

	Arguments:
	    amp: amplitude of the cosine function
	    period: period of the cosine function
	    x0: initial x of the time series
	    xn: final x of the time series
	    step: step of the time series discretization
	    k: exponential rate
	"""
	cos = np.zeros(((xn - x0) * step, 1, 1), dtype=K.floatx())
	for i in range(len(cos)):
		idx = x0 + i * step
		cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
		cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
	return cos 
Example 16
Project: keras_bn_library   Author: bnsnapper   File: layers.py    MIT License 6 votes vote down vote up
def call(self, x, mask=None):
		if self.mode == 'maximum_likelihood':
			# draw maximum likelihood sample from Bernoulli distribution
			#    x* = argmax_x p(x) = 1         if p(x=1) >= 0.5
			#                         0         otherwise
			return K.round(x)
		elif self.mode == 'random':
			# draw random sample from Bernoulli distribution
			#    x* = x ~ p(x) = 1              if p(x=1) > uniform(0, 1)
			#                    0              otherwise
			#return self.srng.binomial(size=x.shape, n=1, p=x, dtype=K.floatx())
			return K.random_binomial(x.shape, p=x, dtype=K.floatx())
		elif self.mode == 'mean_field':
			# draw mean-field approximation sample from Bernoulli distribution
			#    x* = E[p(x)] = E[Bern(x; p)] = p
			return x
		elif self.mode == 'nrlu':
			return nrlu(x)
		else:
			raise NotImplementedError('Unknown sample mode!') 
Example 17
Project: EfficientUnet   Author: zhoudaxia233   File: utils.py    MIT License 6 votes vote down vote up
def conv_kernel_initializer(shape, dtype=K.floatx()):
    """Initialization for convolutional kernels.
    The main difference with tf.variance_scaling_initializer is that
    tf.variance_scaling_initializer uses a truncated normal with an uncorrected
    standard deviation, whereas here we use a normal distribution. Similarly,
    tf.contrib.layers.variance_scaling_initializer uses a truncated normal with
    a corrected standard deviation.
    Args:
        shape: shape of variable
        dtype: dtype of variable
    Returns:
        an initialization for the variable
    """
    kernel_height, kernel_width, _, out_filters = shape
    fan_out = int(kernel_height * kernel_width * out_filters)
    return tf.random_normal(
        shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype) 
Example 18
Project: yoctol-keras-layer-zoo   Author: Yoctol   File: mask_to_seq.py    GNU General Public License v3.0 6 votes vote down vote up
def call(self, inputs, mask=None):
        input_shape = K.int_shape(inputs)
        outputs = self.layer.call(inputs)
        outputs = K.permute_dimensions(
            outputs,
            self.permute_pattern + [len(input_shape) - 1]
        )
        outputs_shape = self.compute_output_shape(input_shape)
        outputs = K.reshape(
            outputs,
            (-1, outputs_shape[1], outputs_shape[2])
        )

        mask_tensor = self.compute_mask(
            inputs,
            mask
        )
        mask_tensor = K.cast(mask_tensor, K.floatx())
        mask_tensor = K.expand_dims(mask_tensor)
        mask_output = K.repeat_elements(
            mask_tensor,
            outputs_shape[2],
            2
        )
        return outputs * mask_output 
Example 19
Project: keras_mixnets   Author: titu1994   File: custom_objects.py    MIT License 5 votes vote down vote up
def __call__(self, shape, dtype=None):
        dtype = dtype or K.floatx()

        kernel_height, kernel_width, _, out_filters = shape
        fan_out = int(kernel_height * kernel_width * out_filters)
        return tf.random_normal(
            shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype)


# Obtained from https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py 
Example 20
Project: keras_mixnets   Author: titu1994   File: custom_objects.py    MIT License 5 votes vote down vote up
def __call__(self, shape, dtype=None):
        dtype = dtype or K.floatx()

        init_range = 1.0 / np.sqrt(shape[1])
        return tf.random_uniform(shape, -init_range, init_range, dtype=dtype)


# Obtained from https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py 
Example 21
Project: phoneticSimilarity   Author: ronggong   File: attentionWithContext.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)
        ait = dot_product(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        result = K.sum(weighted_input, axis=1)

        if self.return_attention:
            return [result, a]
        return result 
Example 22
Project: phoneticSimilarity   Author: ronggong   File: attention.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def call(self, x, mask=None):
        eij = dot_product(x, self.W)  # (samples, steps)

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a_expand = K.expand_dims(a)

        # element wise
        weighted_input = x * a_expand

        result = K.sum(weighted_input, axis=1)

        if self.return_attention:
            return [result, a]
        return result 
Example 23
Project: cdc   Author: ckbjimmy   File: Attention.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        uit = K.dot(x, self.W)
        
        if self.bias:
            uit += self.b
        
        uit = K.tanh(uit)
        
        mul_a = uit  * self.u # with this
        ait = K.sum(mul_a, axis=2) # and this
        
        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1) 
Example 24
Project: AI_Competition   Author: Decalogue   File: attention.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        input_shape = K.int_shape(x)

        features_dim = self.features_dim
        # step_dim = self.step_dim
        step_dim = input_shape[1]

        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))

        if self.bias:
            eij += self.b[:input_shape[1]]

        eij = K.tanh(eij)

        a = K.exp(eij)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
    	# print weigthted_input.shape
        return K.sum(weighted_input, axis=1) 
Example 25
Project: AI_Competition   Author: Decalogue   File: attention.py    MIT License 5 votes vote down vote up
def call(self, inputs, mask=None):
        en = inputs[0]
        de = inputs[1]
        de_shape = K.int_shape(de)
        step_dim = de_shape[1]

        hid_en = K.dot(en, self.W_en1)
        hid_de = K.dot(de, self.W_en2)
        if self.bias:
            hid_en += self.b_en1
            hid_de += self.b_en2
        hid = K.tanh(K.expand_dims(hid_en, axis=1) + hid_de)
        eij = K.reshape(K.dot(hid, K.reshape(self.W_de, (self.hid_size, 1))), (-1, step_dim))
        if self.bias:
            eij += self.b_de[:step_dim]

        a = K.exp(eij - K.max(eij, axis=-1, keepdims=True))

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask[1], K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = de * a
        return K.sum(weighted_input, axis=1) 
Example 26
Project: kaggle-carvana-2017   Author: killthekitten   File: random_transform_mask.py    MIT License 5 votes vote down vote up
def mask_pred(self, batch_x, filenames, index_array, aug=False):
        mask_pred = np.zeros((len(batch_x), self.out_size[0], self.out_size[1], 1), dtype=K.floatx())
        mask_pred[:, :, :, :] = 0.
        for i, j in enumerate(index_array):
            fname = filenames[j]
            mask = self.mask_dir + "/" + fname.split('/')[-1].replace(".jpg", self.mask_suffix)
            mask_pred[i, :, :, :] = img_to_array(
                load_img(mask, grayscale=True, target_size=(self.out_size[0], self.out_size[1]))) / 255.
            if aug:
                batch_x[i, :, :, :], mask_pred[i, :, :, :] = self.random_transform(x=batch_x[i, :, :, :],
                                                                                   mask=mask_pred[i, :, :, :],
                                                                                   height_shift_range=0.0,
                                                                                   width_shift_range=0.0,
                                                                                   shear_range=0.0,
                                                                                   rotation_range=0,
                                                                                   zoom_range=[0.95, 1.05],
                                                                                   channel_shift_range=0.1,
                                                                                   horizontal_flip=True)
        if self.crop_size:
            height = self.crop_size[0]
            width = self.crop_size[1]
            ori_height = self.out_size[0]
            ori_width = self.out_size[1]
            if aug:
                h_start = random.randint(0, ori_height - height - 1)
                w_start = random.randint(0, ori_width - width - 1)
            else:
                # validate on center crops
                h_start = (ori_height - height) // 2
                w_start = (ori_width - width) // 2
            MASK_CROP = mask_pred[:, h_start:h_start + height, w_start:w_start + width, :]
            return batch_x[:, h_start:h_start + height, w_start:w_start + width, :], MASK_CROP
        else:
            return batch_x, mask_pred 
Example 27
Project: dense_tensor   Author: bstriner   File: dense_tensor.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
        assert len(input_shape) == 2
        input_dim = input_shape[1]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_dim))]

        self.W = add_weight(layer=self,
                            shape=(input_dim, self.units),
                            name='{}_W'.format(self.name),
                            initializer=self.kernel_initializer,
                            regularizer=self.kernel_regularizer,
                            constraint=self.kernel_constraint)
        self.V_weights, self.V = self.factorization(name='{}_V'.format(self.name),
                                                    layer=self,
                                                    input_dim=input_dim,
                                                    units=self.units)
        if self.bias:
            self.b = add_weight(layer=self,
                                shape=(self.units,),
                                name='{}_b'.format(self.name),
                                initializer=self.bias_initializer,
                                regularizer=self.bias_regularizer,
                                constraint=self.bias_constraint)

        add_activity_regularizer(layer=self)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
        del self.initial_weights 
Example 28
Project: keras-lamb   Author: CyberZHG   File: optimizer.py    MIT License 5 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1
        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * t))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            mhat_t = m_t / (1. - K.pow(self.beta_1, t))
            vhat_t = v_t / (1. - K.pow(self.beta_2, t))

            u_t = mhat_t / K.sqrt(vhat_t + self.epsilon) + self.weight_decay * p
            trust_ratio = K.sqrt(K.sum(K.square(p)) / K.sum(K.square(u_t)))
            trust_ratio = K.minimum(K.maximum(trust_ratio, self.lower_bound), self.upper_bound)

            lr_p = trust_ratio * lr
            new_p = p - lr_p * u_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 29
Project: cbc_networks   Author: saralajew   File: detection_probability_functions.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        def sqrt(x):
            return K.sqrt(K.maximum(x, K.epsilon()))

        # Both components and input given
        if isinstance(inputs, list) and len(inputs) > 1:
            signals, kernel = inputs
        else:
            signals = inputs
            kernel = self.components.astype(K.floatx())

        # move component_number to channel dimension
        kernel = K.permute_dimensions(kernel, (1, 2, 3, 0))
        # normalize kernel
        normed_kernel = kernel / sqrt(K.sum(K.square(kernel),
                                            (0, 1, 2),
                                            keepdims=True))

        # get norm of signals
        signals_norm = sqrt(K.conv2d(K.square(signals),
                                     np.ones(K.int_shape(kernel)[:3] + (1,),
                                             dtype=K.floatx()),
                                     strides=self.strides,
                                     padding=self.padding,
                                     data_format='channels_last',
                                     dilation_rate=self.dilation_rate))

        diss = K.conv2d(signals,
                        normed_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format='channels_last',
                        dilation_rate=self.dilation_rate) / signals_norm

        if self.n_replicas != 1:
            shape = K.int_shape(diss)
            diss = K.reshape(diss, (-1, shape[1], shape[2],
                                    shape[3] // self.n_replicas,
                                    self.n_replicas))

        return self.activation(diss) 
Example 30
Project: cbc_networks   Author: saralajew   File: detection_probability_functions.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def call(self, inputs, **kwargs):
        # Both components and input given
        if isinstance(inputs, list) and len(inputs) > 1:
            signals, kernel = inputs
        else:
            signals = inputs
            kernel = self.components.astype(K.floatx())

        # move component_number to channel dimension
        kernel = K.permute_dimensions(kernel, (1, 2, 3, 0))
        # normalize kernel
        kernel_norm = K.sum(K.square(kernel), (0, 1, 2), keepdims=True)

        # get norm of signals
        signals_norm = K.conv2d(K.square(signals),
                                K.ones_like(kernel),
                                strides=self.strides,
                                padding=self.padding,
                                data_format='channels_last',
                                dilation_rate=self.dilation_rate)

        diss = kernel_norm \
               - 2 * K.conv2d(signals,
                              kernel,
                              strides=self.strides,
                              padding=self.padding,
                              data_format='channels_last',
                              dilation_rate=self.dilation_rate) \
               + signals_norm

        if self.n_replicas != 1:
            shape = K.int_shape(diss)
            diss = K.reshape(diss, (-1, shape[1], shape[2],
                                    shape[3] // self.n_replicas,
                                    self.n_replicas))

        return self.activation(diss) 
Example 31
Project: Keras_MedicalImgAI   Author: taoyilee   File: chexnet_loss.py    MIT License 5 votes vote down vote up
def class_weights(self):
        for i in range(len(self.class_names)):
            pos_count = np.sum(self.class_labels[..., i], dtype=K.floatx())
            self.wn[i] = pos_count / self.total_count
            self.wp[i] = 1 - self.wn[i]
            print(" [{:>2}] {:>18} W+: {:.3f} W-: {:.3f} W+/W-: {:>7.3f}".format(i + 1, self.class_names[i], self.wp[i],
                                                                                 self.wn[i], self.wp[i] / self.wn[i])) 
Example 32
Project: Keras_MedicalImgAI   Author: taoyilee   File: test_DataSet.py    MIT License 5 votes vote down vote up
def main(config_file):
    tr = Trainer(config_file)
    tr.prepare_datasets()
    tr.prepare_loss_function()
    targets = np.array(tr.train_generator.targets())
    targets = targets[0:3]
    # pred = np.random.rand(14)
    pred = 0.3 * np.ones((3, 14), dtype=K.floatx())
    loss = tr.loss_function(targets, pred)
    print(K.eval(loss)) 
Example 33
Project: ICASSP2019_TCN   Author: DSIP-UPatras   File: custom_layers.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        if mask is not None:
            mask = K.cast(mask, 'float32')
            return K.cast(K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims=True),
                          K.floatx())
        else:
            return K.mean(x, axis=1) 
Example 34
Project: ICASSP2019_TCN   Author: DSIP-UPatras   File: custom_layers.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)
        ait = dot_product(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        result = K.sum(weighted_input, axis=1)

        if self.return_attention:
            return [result, a]
        return result 
Example 35
Project: ICASSP2019_TCN   Author: DSIP-UPatras   File: custom_layers.py    MIT License 5 votes vote down vote up
def call(self, inputs, mask=None):
        if mask is not None:
            if K.ndim(mask) == K.ndim(inputs) - 1:
                mask = K.expand_dims(mask)            

            inputs *= K.cast(mask, K.floatx())

        output = K.conv1d(
            inputs, self.kernel, 
            strides=self.strides[0],
            padding=self.padding,
            data_format=self.data_format,
            dilation_rate=self.dilation_rate[0]
        )

        if self.use_bias:
            m = K.not_equal(output, 0)
            output = K.bias_add(
                output,
                self.bias,
                data_format=self.data_format)
            output *= K.cast(m, K.floatx())
                
        # Apply activations on the image
        if self.activation is not None:
            output = self.activation(output)

        return output 
Example 36
Project: lmtc-eurlex57k   Author: iliaschalkidis   File: pooling.py    Apache License 2.0 5 votes vote down vote up
def call(self, x, mask=None):
        if mask is not None:
            # mask (batch, time)
            mask = K.cast(mask, K.floatx())
            # mask (batch, x_dim, time)
            mask = K.repeat(mask, x.shape[-1])
            # mask (batch, time, x_dim)
            mask = K.tf.transpose(mask, [0, 2, 1])
            x = x * mask
        return K.sum(x, axis=1) / K.sum(mask, axis=1) 
Example 37
Project: lmtc-eurlex57k   Author: iliaschalkidis   File: attention.py    Apache License 2.0 5 votes vote down vote up
def call(self, x, mask=None):
        eij = dot_product(x, self.W)

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        result = K.sum(weighted_input, axis=1)

        if self.return_attention:
            return [result, a]
        return result 
Example 38
Project: lmtc-eurlex57k   Author: iliaschalkidis   File: attention.py    Apache License 2.0 5 votes vote down vote up
def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)

        # Dot product with context vector U
        ait = dot_product(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        result = K.sum(weighted_input, axis=1)

        if self.return_attention:
            return [result, a]
        return result 
Example 39
Project: DisplaceNet   Author: GKalliatakis   File: generic_utils.py    MIT License 5 votes vote down vote up
def weighted_euclidean_distance(self, y_true, y_pred):
        nb_cl = len(self.weights)
        final_mask = K.zeros_like(y_pred[..., 0])
        y_pred_max = K.max(y_pred, axis=-1)
        y_pred_max = K.expand_dims(y_pred_max, axis=-1)
        y_pred_max_mat = K.equal(y_pred, y_pred_max)
        for c_p, c_t in itertools.product(range(nb_cl), range(nb_cl)):
            w = K.cast(self.weights[c_t, c_p], K.floatx())
            y_p = K.cast(y_pred_max_mat[..., c_p], K.floatx())
            y_t = K.cast(y_pred_max_mat[..., c_t], K.floatx())
            final_mask += w * y_p * y_t
        return K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1)) * final_mask 
Example 40
Project: DisplaceNet   Author: GKalliatakis   File: generic_utils.py    MIT License 5 votes vote down vote up
def weighted_binary_crossentropy(self, y_true, y_pred):
        nb_cl = len(self.weights)
        final_mask = K.zeros_like(y_pred[..., 0])
        y_pred_max = K.max(y_pred, axis=-1)
        y_pred_max = K.expand_dims(y_pred_max, axis=-1)
        y_pred_max_mat = K.equal(y_pred, y_pred_max)
        for c_p, c_t in itertools.product(range(nb_cl), range(nb_cl)):
            w = K.cast(self.weights[c_t, c_p], K.floatx())
            y_p = K.cast(y_pred_max_mat[..., c_p], K.floatx())
            y_t = K.cast(y_pred_max_mat[..., c_t], K.floatx())
            final_mask += w * y_p * y_t
        return K.mean(K.binary_crossentropy(y_true, y_pred), axis=-1) * final_mask 
Example 41
Project: dialectal_arabic_segmenter   Author: qcri   File: ChainCRF.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def path_energy0(y, x, U, mask=None):
    '''Path energy without boundary potential handling.'''
    n_classes = K.shape(x)[2]
    y_one_hot = K.one_hot(y, n_classes)

    # Tag path energy
    energy = K.sum(x * y_one_hot, 2)
    energy = K.sum(energy, 1)

    # Transition energy
    y_t = y[:, :-1]
    y_tp1 = y[:, 1:]
    U_flat = K.reshape(U, [-1])
    # Convert 2-dim indices (y_t, y_tp1) of U to 1-dim indices of U_flat:
    flat_indices = y_t * n_classes + y_tp1
    U_y_t_tp1 = K.gather(U_flat, flat_indices)

    if mask is not None:
        mask = K.cast(mask, K.floatx())
        y_t_mask = mask[:, :-1]
        y_tp1_mask = mask[:, 1:]
        U_y_t_tp1 *= y_t_mask * y_tp1_mask

    energy += K.sum(U_y_t_tp1, axis=1)

    return energy 
Example 42
Project: dialectal_arabic_segmenter   Author: qcri   File: ChainCRF.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def viterbi_decode(x, U, b_start=None, b_end=None, mask=None):
    '''Computes the best tag sequence y for a given input x, i.e. the one that
    maximizes the value of path_energy.'''
    x = add_boundary_energy(x, b_start, b_end, mask)

    alpha_0 = x[:, 0, :]
    gamma_0 = K.zeros_like(alpha_0)
    initial_states = [gamma_0, alpha_0]
    _, gamma = _forward(x,
                        lambda B: [K.cast(K.argmax(B, axis=1), K.floatx()), K.max(B, axis=1)],
                        initial_states,
                        U,
                        mask)
    y = _backward(gamma, mask)
    return y 
Example 43
Project: dialectal_arabic_segmenter   Author: qcri   File: ChainCRF.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def build(self, input_shape):
        assert len(input_shape) == 3
        n_classes = input_shape[2]
        n_steps = input_shape[1]
        assert n_classes >= 2
        assert n_steps is None or n_steps >= 2
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, n_steps, n_classes))]

        self.U = self.add_weight((n_classes, n_classes),
                                 initializer=self.init,
                                 name='{}_U'.format(self.name),
                                 regularizer=self.U_regularizer,
                                 constraint=self.U_constraint)

        self.b_start = self.add_weight((n_classes, ),
                                       initializer='zero',
                                       name='{}_b_start'.format(self.name),
                                       regularizer=self.b_start_regularizer,
                                       constraint=self.b_start_constraint)

        self.b_end = self.add_weight((n_classes, ),
                                     initializer='zero',
                                     name='{}_b_end'.format(self.name),
                                     regularizer=self.b_end_regularizer,
                                     constraint=self.b_end_constraint)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

        self.built = True 
Example 44
Project: musical_genres_classification   Author: shaoeric   File: Attention.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        features_dim = self.features_dim
        # 这里应该是 step_dim是我们指定的参数,它等于input_shape[1],也就是rnn的timesteps
        step_dim = self.step_dim

        # 输入和参数分别reshape再点乘后,tensor.shape变成了(batch_size*timesteps, 1),之后每个batch要分开进行归一化
		# 所以应该有 eij = K.reshape(..., (-1, timesteps))
        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))

        if self.bias:
            eij += self.b
        # RNN一般默认激活函数为tanh, 对attention来说激活函数差别不打,因为要做softmax
        eij = K.tanh(eij)

        a = K.exp(eij)

        if mask is not None:
        # 如果前面的层有mask,那么后面这些被mask掉的timestep肯定是不能参与计算输出的,也就是将他们的attention权重设为0
            a *= K.cast(mask, K.floatx())
        # cast是做类型转换,keras计算时会检查类型,可能是因为用gpu的原因
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        # a = K.expand_dims(a, axis=-1) , axis默认为-1, 表示在最后扩充一个维度。
		# 比如shape = (3,)变成 (3, 1)
        a = K.expand_dims(a)
        # 此时a.shape = (batch_size, timesteps, 1), x.shape = (batch_size, timesteps, units)
        weighted_input = x * a

        # weighted_input的shape为 (batch_size, timesteps, units), 每个timestep的输出向量已经乘上了该timestep的权重
		# weighted_input在axis=1上取和,返回值的shape为 (batch_size, 1, units)
        return K.sum(weighted_input, axis=1) 
Example 45
Project: musical_genres_classification   Author: shaoeric   File: Attention.py    MIT License 5 votes vote down vote up
def call(self, x, mask=None):
        features_dim = self.features_dim
        # 这里应该是 step_dim是我们指定的参数,它等于input_shape[1],也就是rnn的timesteps
        step_dim = self.step_dim

        # 输入和参数分别reshape再点乘后,tensor.shape变成了(batch_size*timesteps, 1),之后每个batch要分开进行归一化
		# 所以应该有 eij = K.reshape(..., (-1, timesteps))
        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))

        if self.bias:
            eij += self.b
        # RNN一般默认激活函数为tanh, 对attention来说激活函数差别不打,因为要做softmax
        eij = K.tanh(eij)

        a = K.exp(eij)

        if mask is not None:
        # 如果前面的层有mask,那么后面这些被mask掉的timestep肯定是不能参与计算输出的,也就是将他们的attention权重设为0
            a *= K.cast(mask, K.floatx())
        # cast是做类型转换,keras计算时会检查类型,可能是因为用gpu的原因
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        # a = K.expand_dims(a, axis=-1) , axis默认为-1, 表示在最后扩充一个维度。
		# 比如shape = (3,)变成 (3, 1)
        a = K.expand_dims(a)
        # 此时a.shape = (batch_size, timesteps, 1), x.shape = (batch_size, timesteps, units)
        weighted_input = x * a

        # weighted_input的shape为 (batch_size, timesteps, units), 每个timestep的输出向量已经乘上了该timestep的权重
		# weighted_input在axis=1上取和,返回值的shape为 (batch_size, 1, units)
        return K.sum(weighted_input, axis=1) 
Example 46
Project: keras_nade   Author: jgrnt   File: utils.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __call__(self, x):
        return K.sum(K.exp(1 * (K.relu(self.min - x) + K.relu(x - self.max)))) - K.prod(K.cast(K.shape(x), K.floatx())) 
Example 47
Project: keras_nade   Author: jgrnt   File: orderless_nade.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def call(self, x, **kwargs):
        from theano import tensor as T
        from theano.tensor.shared_randomstreams import RandomStreams
        if K.backend() == "theano":
            import theano
            mask_rng = RandomStreams(self.seed)

            ints = mask_rng.random_integers(size=K.expand_dims(x.shape[0], 0), high=x.shape[1] - 1)

            def set_value_at_position(i, ns_x):
                zeros = T.zeros_like(ns_x[0, :])
                return T.set_subtensor(zeros[:i], 1)

            result, updates = theano.scan(fn=set_value_at_position,
                                          outputs_info=None,
                                          sequences=ints,
                                          non_sequences=x)
            mask = mask_rng.shuffle_row_elements(result)
        elif K.backend() == "tensorflow":
            import tensorflow as tf
            tf.set_random_seed(self.seed)
            ints = tf.random_uniform(shape=K.expand_dims(tf.shape(x)[0], 0),
                                     maxval=x.shape[1],
                                     dtype=tf.int32)
            result = tf.sequence_mask(ints, maxlen=x.shape[1])
            parallel_iterations = self._deterministic and 1 or 10
            mask = tf.cast(tf.map_fn(tf.random_shuffle, result, parallel_iterations=parallel_iterations), K.floatx())
        else:
            raise NotImplementedError()
        return K.concatenate([x * mask, mask]) 
Example 48
Project: KDDCup2019_admin   Author: DominickZhang   File: fm_keras.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
        assert len(input_shape) == 2
        input_dim = input_shape[1]
        numeric_size = input_dim - self.feature_num
        self.numeric_size = numeric_size
        all_size = numeric_size + self.feature_size

        self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim))

        self.w_one_hot = self.add_weight(name='one_one_hot', 
                                 shape=(self.feature_size, self.output_dim),
                                 initializer='glorot_uniform',
                                 trainable=True)
        self.w_numeric = self.add_weight(name='one_numeric', 
                                 shape=(numeric_size, self.output_dim),
                                 initializer='glorot_uniform',
                                 trainable=True)
        self.v_one_hot = self.add_weight(name='two_one_hot', 
                                 shape=(self.feature_size, self.embedding_size),
                                 initializer='glorot_uniform',
                                 trainable=True)
        self.v_numeric = self.add_weight(name='two_numeric', 
                                 shape=(numeric_size, self.embedding_size),
                                 initializer='glorot_uniform',
                                 trainable=True)
        self.b = self.add_weight(name='bias', 
                                 shape=(self.output_dim,),
                                 initializer='zeros',
                                 trainable=True)

        super(FMLayer, self).build(input_shape) 
Example 49
Project: fcd_torch   Author: insilicomedicine   File: keras2pytorch.py    MIT License 5 votes vote down vote up
def masked_accuracy(y_true, y_pred):
    mask_value = 0.5
    a = K.sum(K.cast(K.equal(y_true, K.round(y_pred)), K.floatx()))
    c = K.sum(K.cast(K.not_equal(y_true, mask_value), K.floatx()))
    acc = (a) / c
    return acc 
Example 50
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 5 votes vote down vote up
def _get_gamma(self, feature_dim):
        """Get the number of activation units to drop"""
        feature_dim = K.cast(feature_dim, K.floatx())
        block_size = K.constant(self.block_size, dtype=K.floatx())
        return ((1.0 - self.keep_prob) / block_size) * (feature_dim / (feature_dim - block_size + 1.0)) 
Example 51
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 5 votes vote down vote up
def _get_gamma(self, height, width):
        """Get the number of activation units to drop"""
        height, width = K.cast(height, K.floatx()), K.cast(width, K.floatx())
        block_size = K.constant(self.block_size, dtype=K.floatx())
        return ((1.0 - self.keep_prob) / (block_size ** 2)) *\
               (height * width / ((height - block_size + 1.0) * (width - block_size + 1.0))) 
Example 52
Project: YouTubeCommenter   Author: HackerPoet   File: Generate.py    MIT License 5 votes vote down vote up
def new_sparse_categorical_accuracy(y_true, y_pred):
	y_pred_rank = ops.convert_to_tensor(y_pred).get_shape().ndims
	y_true_rank = ops.convert_to_tensor(y_true).get_shape().ndims
	# If the shape of y_true is (num_samples, 1), squeeze to (num_samples,)
	if (y_true_rank is not None) and (y_pred_rank is not None) and (len(K.int_shape(y_true)) == len(K.int_shape(y_pred))):
		y_true = array_ops.squeeze(y_true, [-1])
	y_pred = math_ops.argmax(y_pred, axis=-1)
	# If the predicted output and actual output types don't match, force cast them
	# to match.
	if K.dtype(y_pred) != K.dtype(y_true):
		y_pred = math_ops.cast(y_pred, K.dtype(y_true))
	return math_ops.cast(math_ops.equal(y_true, y_pred), K.floatx())

#Load the model 
Example 53
Project: YouTubeCommenter   Author: HackerPoet   File: Train.py    MIT License 5 votes vote down vote up
def new_sparse_categorical_accuracy(y_true, y_pred):
	y_pred_rank = ops.convert_to_tensor(y_pred).get_shape().ndims
	y_true_rank = ops.convert_to_tensor(y_true).get_shape().ndims
	# If the shape of y_true is (num_samples, 1), squeeze to (num_samples,)
	if (y_true_rank is not None) and (y_pred_rank is not None) and (len(K.int_shape(y_true)) == len(K.int_shape(y_pred))):
		y_true = array_ops.squeeze(y_true, [-1])
	y_pred = math_ops.argmax(y_pred, axis=-1)
	# If the predicted output and actual output types don't match, force cast them
	# to match.
	if K.dtype(y_pred) != K.dtype(y_true):
		y_pred = math_ops.cast(y_pred, K.dtype(y_true))
	return math_ops.cast(math_ops.equal(y_true, y_pred), K.floatx())

#Build the training models 
Example 54
Project: humpback-whale-4th-place   Author: daustingm1   File: ensemble.py    Apache License 2.0 5 votes vote down vote up
def score_reshape(score, x, y=None):
    """
    Tranformed the packed matrix 'score' into a square matrix.
    @param score the packed matrix
    @param x the first image feature tensor
    @param y the second image feature tensor if different from x
    @result the square matrix
    """
    if y is None:
        # When y is None, score is a packed upper triangular matrix.
        # Unpack, and transpose to form the symmetrical lower triangular matrix.
        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx()) 
Example 55
Project: humpback-whale-4th-place   Author: daustingm1   File: ensemble.py    Apache License 2.0 5 votes vote down vote up
def score_reshapeV(scoreV, x, y=None):
    """
    Tranformed the packed matrix 'score' into a square matrix.
    @param score the packed matrix
    @param x the first image feature tensor
    @param y the second image feature tensor if different from x
    @result the square matrix
    """
    if y is None:
        # When y is None, score is a packed upper triangular matrix.
        # Unpack, and transpose to form the symmetrical lower triangular matrix.
        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx()) 
Example 56
Project: humpback-whale-4th-place   Author: daustingm1   File: siamese_train.py    Apache License 2.0 5 votes vote down vote up
def score_reshape(score, x, y=None):
    """
    Tranformed the packed matrix 'score' into a square matrix.
    @param score the packed matrix
    @param x the first image feature tensor
    @param y the second image feature tensor if different from x
    @result the square matrix
    """
    if y is None:
        # When y is None, score is a packed upper triangular matrix.
        # Unpack, and transpose to form the symmetrical lower triangular matrix.
        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx()) 
Example 57
Project: humpback-whale-4th-place   Author: daustingm1   File: siamese_inference.py    Apache License 2.0 5 votes vote down vote up
def score_reshape(score, x, y=None):
    """
    Tranformed the packed matrix 'score' into a square matrix.
    @param score the packed matrix
    @param x the first image feature tensor
    @param y the second image feature tensor if different from x
    @result the square matrix
    """
    if y is None:
        # When y is None, score is a packed upper triangular matrix.
        # Unpack, and transpose to form the symmetrical lower triangular matrix.
        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx()) 
Example 58
Project: humpback-whale-4th-place   Author: daustingm1   File: siamese_inference.py    Apache License 2.0 5 votes vote down vote up
def score_reshapeV(scoreV, x, y=None):
    """
    Tranformed the packed matrix 'score' into a square matrix.
    @param score the packed matrix
    @param x the first image feature tensor
    @param y the second image feature tensor if different from x
    @result the square matrix
    """
    if y is None:
        # When y is None, score is a packed upper triangular matrix.
        # Unpack, and transpose to form the symmetrical lower triangular matrix.
        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx()) 
Example 59
Project: deepGroupv2   Author: albu5   File: networks.py    MIT License 5 votes vote down vote up
def my_categorical_accuracy(y_true, y_pred):
    y_true = kb.argmax(y_true, axis=2)
    y_pred = kb.argmax(y_pred, axis=2)
    return tf.reduce_mean(tf.cast(y_true == y_pred, kb.floatx()), axis=None) 
Example 60
Project: deepGroupv2   Author: albu5   File: networks.py    MIT License 5 votes vote down vote up
def my_categorical_crossentropy_mask(target, output, from_logits=False):
    """Categorical crossentropy between an output tensor and a target tensor.

    # Arguments
        target: A tensor of the same shape as `output`.
        output: A tensor resulting from a softmax
            (unless `from_logits` is True, in which
            case `output` is expected to be the logits).
        from_logits: Boolean, whether `output` is the
            result of a softmax, or is a tensor of logits.

    # Returns
        Output tensor.
    """
    # Note: tf.nn.softmax_cross_entropy_with_logits
    # expects logits, Keras expects probabilities.
    if not from_logits:
        # scale preds so that the class probas of each sample sum to 1
        output /= tf.reduce_sum(output,
                                len(output.get_shape()) - 1,
                                True)
        # manual computation of crossentropy
        _epsilon = tf.convert_to_tensor(kb.epsilon(), output.dtype.base_dtype)
        output = tf.clip_by_value(output, _epsilon, 1. - _epsilon)
        valid_logs = tf.reduce_sum(target * tf.log(output), axis=3)
        valid_logs *= tf.cast(tf.argmax(target, axis=3) > 0, dtype=kb.floatx())
        return -tf.reduce_sum(valid_logs)
    else:
        return tf.nn.softmax_cross_entropy_with_logits(labels=target,
                                                       logits=output) 
Example 61
Project: deepGroupv2   Author: albu5   File: networks.py    MIT License 5 votes vote down vote up
def my_categorical_accuracy_mask(y_true, y_pred):
    y_true = tf.argmax(y_true, axis=3)
    y_pred = tf.argmax(y_pred, axis=3)
    correct = tf.cast(y_true == y_pred, dtype=kb.floatx())
    correct *= tf.cast(y_true > 0, dtype=kb.floatx())
    return tf.reduce_sum(tf.cast(correct, kb.floatx())) / tf.reduce_sum(tf.cast(y_true > 0, dtype=kb.floatx())) 
Example 62
Project: zlyang-seq2seq-gec   Author: young-zonglin   File: basic_model.py    MIT License 5 votes vote down vote up
def _masked_categorical_accuracy(self, target, preds):
        y_mask = GetPadMask(self.batch_size)(target)
        raw_tag = K.cast(K.equal(K.argmax(target, axis=-1),
                                 K.argmax(preds, axis=-1)),
                         K.floatx())
        assert K.ndim(raw_tag) == 2
        return raw_tag * y_mask

    # TODO 优化算法
    # 动态学习率 => done,在回调中更改学习率
    # Get done => Masked loss function. 
Example 63
Project: BERT   Author: yyht   File: funcs.py    Apache License 2.0 5 votes vote down vote up
def scaled_dot_product_attention_tf(q, k, v, attn_mask, attention_dropout: float):
    w = K.batch_dot(q, k)  # w is B, H, L, L
    w = w / K.sqrt(K.cast(shape_list(v)[-1], K.floatx()))
    if attn_mask is not None:
        w = attn_mask * w + (1.0 - attn_mask) * -1e9
    w = K.softmax(w)
    w = Dropout(attention_dropout)(w)
    return K.batch_dot(w, v)  # it is B, H, L, C//H [like v] 
Example 64
Project: BERT   Author: yyht   File: funcs.py    Apache License 2.0 5 votes vote down vote up
def scaled_dot_product_attention_th(q, k, v, attn_mask, attention_dropout: float):
    w = theano_matmul(q, k)
    w = w / K.sqrt(K.cast(shape_list(v)[-1], K.floatx()))
    if attn_mask is not None:
        attn_mask = K.repeat_elements(attn_mask, shape_list(v)[1], 1)
        w = attn_mask * w + (1.0 - attn_mask) * -1e9
    w = K.T.exp(w - w.max()) / K.T.exp(w - w.max()).sum(axis=-1, keepdims=True)
    w = Dropout(attention_dropout)(w)
    return theano_matmul(w, v) 
Example 65
Project: BERT   Author: yyht   File: train.py    Apache License 2.0 5 votes vote down vote up
def _mask_loss(y_true, y_pred, y_mask, element_wise_loss):
    l = K.switch(y_mask, element_wise_loss(y_true, y_pred), K.zeros_like(y_mask, dtype=K.floatx()))
    return K.sum(l) / (K.cast(K.sum(y_mask), dtype='float32') + K.epsilon()) 
Example 66
Project: group-ksparse-temporal-cnns   Author: srph25   File: ops.py    MIT License 5 votes vote down vote up
def group_ksparse(x, groups, k, axis_group, axis_sparse, norm=2, alpha=1, epsilon=None):
    if isinstance(axis_group, int):
        axis_group = (axis_group,)
    elif isinstance(axis_group, list):
        axis_group = tuple(axis_group)
    if isinstance(axis_sparse, int):
        axis_sparse = (axis_sparse,)
    elif isinstance(axis_sparse, list):
        axis_sparse = tuple(axis_sparse)
    assert(1 - bool(set(axis_group) & set(axis_sparse)))
    if epsilon is None:
        epsilon = K.epsilon()
    axis_complement = tuple(set(range(K.ndim(x))) - set(axis_group) - set(axis_sparse))
    shape_reduce_group = K.prod([K.shape(x)[j] for j in axis_group])
    shape_reduce_sparse = K.prod([K.shape(x)[j] for j in axis_sparse])
    _k = K.minimum(K.in_train_phase(k, alpha * k), shape_reduce_sparse)
    inputs_permute_dimensions = K.permute_dimensions(x, axis_complement + axis_sparse + axis_group)
    inputs_permute_dimensions_reshape = K.reshape(inputs_permute_dimensions, (-1, shape_reduce_sparse, shape_reduce_group))
    norm_group_permute_dimensions_reshape = group_norms(inputs=inputs_permute_dimensions_reshape, groups=groups, axis=-1, norm=norm, epsilon=epsilon)
    norm_group_permute_dimensions_reshape = K.permute_dimensions(norm_group_permute_dimensions_reshape, (0, 2, 1))
    norm_group_permute_dimensions_reshape = K.reshape(norm_group_permute_dimensions_reshape, (-1, shape_reduce_sparse))
    _, indices = tf.nn.top_k(norm_group_permute_dimensions_reshape, _k)
    scatter_indices = K.concatenate([(K.arange(K.shape(norm_group_permute_dimensions_reshape)[0])[:, None] * K.ones((1, _k), dtype='int32'))[:, :, None], indices[:, :, None]])
    scatter_updates = K.ones((K.shape(norm_group_permute_dimensions_reshape)[0], _k))
    mask_group_permute_dimensions_reshape = K.cast(tf.scatter_nd(scatter_indices, scatter_updates, K.shape(norm_group_permute_dimensions_reshape)), K.floatx())
    mask_group_permute_dimensions_reshape = K.reshape(mask_group_permute_dimensions_reshape, (-1, groups, shape_reduce_sparse))
    mask_group_permute_dimensions_reshape = K.permute_dimensions(mask_group_permute_dimensions_reshape, (0, 2, 1))
    mask_permute_dimensions_reshape = (mask_group_permute_dimensions_reshape[:, :, :, None] * K.ones((1, 1, 1, floor_div(shape_reduce_group, groups))))
    mask_permute_dimensions = K.reshape(mask_permute_dimensions_reshape, K.shape(inputs_permute_dimensions))
    mask = K.permute_dimensions(mask_permute_dimensions, tuple(np.argsort(axis_complement + axis_sparse + axis_group)))
    return mask * x 
Example 67
Project: group-ksparse-temporal-cnns   Author: srph25   File: keraswtacrnn.py    MIT License 5 votes vote down vote up
def __init__(self, results_dir, config):
        super(KerasWTACRNN, self).__init__(results_dir, config)
        if 'pad_value' in self.config.keys():
            self.loss = lambda true, pred: K.mean(K.square(pred[:, :-1] - true[:, 1:]) * K.cast(K.not_equal(true[:, 1:], self.config['pad_value']), K.floatx()), axis=-1)
        else:
            self.loss = lambda true, pred: K.mean(K.square(pred[:, :-1] - true[:, 1:]), axis=-1) 
Example 68
Project: group-ksparse-temporal-cnns   Author: srph25   File: keraswtacnn.py    MIT License 5 votes vote down vote up
def __init__(self, results_dir, config):
        self.results_dir = results_dir
        self.set_config(config)
        if 'pad_value' in self.config.keys():
            self.loss = lambda true, pred: K.mean(K.square(pred - true) * K.cast(K.not_equal(true, self.config['pad_value']), K.floatx()), axis=-1)
        else:
            self.loss = lambda true, pred: K.mean(K.square(pred - true), axis=-1) 
Example 69
Project: keras_bn_library   Author: bnsnapper   File: rbm.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
		assert len(input_shape) == 2
		input_dim = input_shape[-1]
		self.input_spec = [InputSpec(dtype=K.floatx(),
									ndim='2+')]

		#self.trainable_weights = [self.W, self.bx, self.bh]

		if self.initial_weights is not None:
			self.set_weights(self.initial_weights)
		del self.initial_weights 
Example 70
Project: EfficientUnet   Author: zhoudaxia233   File: utils.py    MIT License 5 votes vote down vote up
def dense_kernel_initializer(shape, dtype=K.floatx()):
    init_range = 1.0 / np.sqrt(shape[1])
    return tf.random_uniform(shape, -init_range, init_range, dtype=dtype) 
Example 71
Project: yoctol-keras-layer-zoo   Author: Yoctol   File: maskConv.py    GNU General Public License v3.0 5 votes vote down vote up
def call(self, inputs):
        inputs_shape = K.int_shape(inputs)
        channel_axis = len(inputs_shape) - 1
        masked_tensor = self.compute_mask(inputs)
        masked_tensor = K.expand_dims(masked_tensor)
        masked_tensor = K.repeat_elements(
            masked_tensor,
            inputs_shape[channel_axis],
            channel_axis
        )
        return inputs * K.cast(masked_tensor, K.floatx()) 
Example 72
Project: kaggle-carvana-2017   Author: killthekitten   File: keras_iterator.py    MIT License 4 votes vote down vote up
def next(self):
        """For python 2.x.

        # Returns
            The next batch.
        """
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        # The transformation of images is not under thread lock
        # so it can be done in parallel
        batch_x = np.zeros((current_batch_size,) + self.image_shape, dtype=K.floatx())
        grayscale = self.color_mode == 'grayscale'
        # build batch of image data
        for i, j in enumerate(index_array):
            fname = self.filenames[j]
            img = load_img(os.path.join(self.directory, fname),
                           grayscale=grayscale,
                           target_size=self.target_size)
            x = img_to_array(img, data_format=self.data_format)
            x = self.image_data_generator.random_transform(x)
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
        # optionally save augmented images to disk for debugging purposes
        if self.save_to_dir:
            for i in range(current_batch_size):
                img = array_to_img(batch_x[i], self.data_format, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=current_index + i,
                                                                  hash=np.random.randint(1e4),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        # build batch of labels
        if self.class_mode == 'sparse':
            batch_y = self.classes[index_array]
        elif self.class_mode == 'binary':
            batch_y = self.classes[index_array].astype(K.floatx())
        elif self.class_mode == 'categorical':
            batch_y = np.zeros((len(batch_x), self.num_class), dtype=K.floatx())
            for i, label in enumerate(self.classes[index_array]):
                batch_y[i, label] = 1.
        elif self.output_function:
            return self.output_function(batch_x, self.filenames, index_array, self.file_name_to_y)
        else:
            return batch_x

        return batch_x, batch_y 
Example 73
Project: bert_lamb_pretrain   Author: goldenbili   File: keras_lamb.py    Apache License 2.0 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            m_t_hat = m_t / (1. - K.pow(self.beta_1, t))
            v_t_hat = v_t / (1. - K.pow(self.beta_2, t))

            p_dash = m_t_hat / (K.sqrt(v_t_hat + self.epsilon))

            if self.weight_decay > 0.:
                wd = self.weight_decay * p
                p_dash = p_dash + wd

            r1 = K.sqrt(K.sum(K.square(p)))
            r2 = K.sqrt(K.sum(K.square(p_dash)))

            r = tf.where(tf.greater(r1, 0.),
                         tf.where(tf.greater(r2, 0.),
                                  r1 / r2,
                                  1.0),
                         1.0)
            # r = r1 / r2
            eta = r * lr

            p_t = p - eta * p_dash

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 74
Project: keras_radam   Author: forin-xyz   File: radam.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay:
            lr = lr * (1. / (1. + self.decay * K.cast(
                self.iterations, K.dtype(self.decay)
            )))

        t = K.cast(self.iterations, K.floatx()) + 1.
        beta_1 = self.beta_1
        beta_2 = self.beta_2
        beta_1_t = K.pow(beta_1, t)
        beta_2_t = K.pow(beta_2, t)
        rho_inf = 2. / (1. - beta_2) - 1.
        rho_t = rho_inf - 2. * t * beta_2_t / (1. - beta_2_t)
        r_t = K.sqrt(
            K.relu(rho_t - 4.) * (rho_t - 2.) * rho_inf / (
                (rho_inf - 4.) * (rho_inf - 2.) * rho_t )
        )
        flag = K.cast(rho_t > 4., K.floatx())

        ms = [K.zeros(K.int_shape(p)) for p in params]
        vs = [K.zeros(K.int_shape(p)) for p in params]

        self.weights = [self.iterations] + ms + vs
        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = beta_1 * m + (1. - beta_1) * g
            v_t = beta_2 * v + (1. - beta_2) * K.square(g)

            m_hat_t = m_t / (1. - beta_1_t)
            v_hat_t = K.sqrt(v_t / (1. - beta_2_t))
            new_p = p - lr * (r_t / (v_hat_t + self.epsilon) + flag - 1.)* m_hat_t

            if getattr(p, "constraint", None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
        return self.updates 
Example 75
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]

        if self.amsgrad:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        else:
            vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = lr_t * self.eta_t  # for external tracking
        lr_t_premult = lr_t

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            # Learning rate multipliers
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t_premult, p)

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            if self.amsgrad:
                vhat_t = K.maximum(vhat, v_t)
                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, vhat_t))
            else:
                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 76
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        t = K.cast(self.iterations, K.floatx()) + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = self.lr * self.eta_t  # for external tracking

        for p, g, m, v in zip(params, grads, ms, vs):
            # Learning rate multipliers
            lr_t = self.lr
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t, p)

            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + (
                momentum_cache_t_1 * m_t_prime)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            p_t = p - self.eta_t * lr_t * m_t_bar / (
                    K.sqrt(v_t_prime) + self.epsilon)

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 77
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 78
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 79
Project: iMIMIC-RCVs   Author: medgift   File: tcav.py    MIT License 4 votes vote down vote up
def visualize_cam_with_losses(input_tensor, losses, seed_input, penultimate_layer, grad_modifier=None):
    """Generates a gradient based class activation map (CAM) by using positive gradients of `input_tensor`
    with respect to weighted `losses`.

    For details on grad-CAM, see the paper:
    [Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via Gradient-based Localization]
    (https://arxiv.org/pdf/1610.02391v1.pdf).

    Unlike [class activation mapping](https://arxiv.org/pdf/1512.04150v1.pdf), which requires minor changes to
    network architecture in some instances, grad-CAM has a more general applicability.

    Compared to saliency maps, grad-CAM is class discriminative; i.e., the 'cat' explanation exclusively highlights
    cat regions and not the 'dog' region and vice-versa.

    Args:
        input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
            channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
        losses: List of ([Loss](vis.losses#Loss), weight) tuples.
        seed_input: The model input for which activation map needs to be visualized.
        penultimate_layer: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
            with respect to filter output.
        grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
            specify anything, gradients are unchanged (Default value = None)

    Returns:
        The normalized gradients of `seed_input` with respect to weighted `losses`.
    """
    penultimate_output = penultimate_layer.output
    opt = Optimizer(input_tensor, losses, wrt_tensor=penultimate_output, norm_grads=False)
    _, grads, penultimate_output_value = opt.minimize(seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)

    # For numerical stability. Very small grad values along with small penultimate_output_value can cause
    # w * penultimate_output_value to zero out, even for reasonable fp precision of float32.
    grads = grads / (np.max(grads) + K.epsilon())

    # Average pooling across all feature maps.
    # This captures the importance of feature map (channel) idx to the output.
    channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
    other_axis = np.delete(np.arange(len(grads.shape)), channel_idx)
    weights = np.mean(grads, axis=tuple(other_axis))

    # Generate heatmap by computing weight * output over feature maps
    output_dims = utils.get_img_shape(penultimate_output)[2:]
    heatmap = np.zeros(shape=output_dims, dtype=K.floatx())
    for i, w in enumerate(weights):
        if channel_idx == -1:
            heatmap += w * penultimate_output_value[0, ..., i]
        else:
            heatmap += w * penultimate_output_value[0, i, ...]

    # ReLU thresholding to exclude pattern mismatch information (negative gradients).
    heatmap = np.maximum(heatmap, 0)

    # The penultimate feature map size is definitely smaller than input image.
    input_dims = utils.get_img_shape(input_tensor)[2:]

    # Figure out the zoom factor.
    zoom_factor = [i / (j * 1.0) for i, j in iter(zip(input_dims, output_dims))]
    heatmap = zoom(heatmap, zoom_factor)
    return utils.normalize(heatmap) 
Example 80
Project: iMIMIC-RCVs   Author: medgift   File: saliency.py    MIT License 4 votes vote down vote up
def visualize_cam_with_losses(input_tensor, losses, seed_input, penultimate_layer, grad_modifier=None):
    """Generates a gradient based class activation map (CAM) by using positive gradients of `input_tensor`
    with respect to weighted `losses`.

    For details on grad-CAM, see the paper:
    [Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via Gradient-based Localization]
    (https://arxiv.org/pdf/1610.02391v1.pdf).

    Unlike [class activation mapping](https://arxiv.org/pdf/1512.04150v1.pdf), which requires minor changes to
    network architecture in some instances, grad-CAM has a more general applicability.

    Compared to saliency maps, grad-CAM is class discriminative; i.e., the 'cat' explanation exclusively highlights
    cat regions and not the 'dog' region and vice-versa.

    Args:
        input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
            channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
        losses: List of ([Loss](vis.losses#Loss), weight) tuples.
        seed_input: The model input for which activation map needs to be visualized.
        penultimate_layer: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
            with respect to filter output.
        grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
            specify anything, gradients are unchanged (Default value = None)

    Returns:
        The normalized gradients of `seed_input` with respect to weighted `losses`.
    """
    penultimate_output = penultimate_layer.output
    opt = Optimizer(input_tensor, losses, wrt_tensor=penultimate_output, norm_grads=False)
    _, grads, penultimate_output_value = opt.minimize(seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)

    # For numerical stability. Very small grad values along with small penultimate_output_value can cause
    # w * penultimate_output_value to zero out, even for reasonable fp precision of float32.
    grads = grads / (np.max(grads) + K.epsilon())

    # Average pooling across all feature maps.
    # This captures the importance of feature map (channel) idx to the output.
    channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
    other_axis = np.delete(np.arange(len(grads.shape)), channel_idx)
    weights = np.mean(grads, axis=tuple(other_axis))

    # Generate heatmap by computing weight * output over feature maps
    output_dims = utils.get_img_shape(penultimate_output)[2:]
    heatmap = np.zeros(shape=output_dims, dtype=K.floatx())
    for i, w in enumerate(weights):
        if channel_idx == -1:
            heatmap += w * penultimate_output_value[0, ..., i]
        else:
            heatmap += w * penultimate_output_value[0, i, ...]

    # ReLU thresholding to exclude pattern mismatch information (negative gradients).
    heatmap = np.maximum(heatmap, 0)

    # The penultimate feature map size is definitely smaller than input image.
    input_dims = utils.get_img_shape(input_tensor)[2:]

    # Figure out the zoom factor.
    zoom_factor = [i / (j * 1.0) for i, j in iter(zip(input_dims, output_dims))]
    heatmap = zoom(heatmap, zoom_factor)
    return utils.normalize(heatmap)