Python keras.backend.zeros() Examples

The following are code examples for showing how to use keras.backend.zeros(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: CapsNet   Author: l11x0m7   File: capsule.py    MIT License 6 votes vote down vote up
def call(self, inputs, **kwargs):
        # (batch_size, 1, input_num_capsule, input_dim_capsule)
        expand_inputs = K.expand_dims(inputs, axis=1)
        # (batch_size, num_capsule, input_num_capsule, input_dim_capsule)
        expand_inputs = K.tile(expand_inputs, (1, self.num_capsule, 1, 1))
        # (batch_size, num_capsule, input_num_capsule, dim_capsule)
        u_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, axes=[2, 3]), expand_inputs)

        if self.num_routing <= 0:
            self.num_routing = 3
        # (batch_size, num_capsule, input_num_capsule)
        b = K.zeros((K.shape(u_hat)[0], self.num_capsule, self.input_num_capsule))
        for i in xrange(self.num_routing):
            # (batch_size, num_capsule, input_num_capsule)
            c = softmax(b, axis=1)
            # (batch_size, num_capsule, dim_capsule)
            s = K.batch_dot(c, u_hat, axes=[2, 2])
            squashed_s = squash(s)
            if i < self.num_routing - 1:
                # (batch_size, num_capsule, input_num_capsule)
                b += K.batch_dot(squashed_s, u_hat, axes=[2, 3])
        return squashed_s 
Example 2
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 6 votes vote down vote up
def build(self, input_shape):
        assert len(input_shape) >= 3
        self.input_spec = [InputSpec(shape=input_shape)]
        nb_samples, nb_time, input_dim = input_shape

        if not self.layer.built:
            self.layer.build(input_shape)
            self.layer.built = True

        super(Attention, self).build()
        
        self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name))
        self.W2 = self.layer.init((self.layer.output_dim, input_dim), name='{}_W2'.format(self.name))
        self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name))
        self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name))
        self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name))
        self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name))

        self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3] 
Example 3
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 6 votes vote down vote up
def _compute_valid_seed_region(self, seq_length):
        positions = K.arange(seq_length)
        half_block_size = self.block_size // 2
        valid_seed_region = K.switch(
            K.all(
                K.stack(
                    [
                        positions >= half_block_size,
                        positions < seq_length - half_block_size,
                    ],
                    axis=-1,
                ),
                axis=-1,
            ),
            K.ones((seq_length,)),
            K.zeros((seq_length,)),
        )
        return K.expand_dims(K.expand_dims(valid_seed_region, axis=0), axis=-1) 
Example 4
Project: Keras-DropBlock   Author: MLearing   File: drop_block.py    MIT License 6 votes vote down vote up
def _compute_valid_seed_region(self, height, width):
        positions = K.concatenate([
            K.expand_dims(K.tile(K.expand_dims(K.arange(height), axis=1), [1, width]), axis=-1),
            K.expand_dims(K.tile(K.expand_dims(K.arange(width), axis=0), [height, 1]), axis=-1),
        ], axis=-1)
        half_block_size = self.block_size // 2
        valid_seed_region = K.switch(
            K.all(
                K.stack(
                    [
                        positions[:, :, 0] >= half_block_size,
                        positions[:, :, 1] >= half_block_size,
                        positions[:, :, 0] < height - half_block_size,
                        positions[:, :, 1] < width - half_block_size,
                    ],
                    axis=-1,
                ),
                axis=-1,
            ),
            K.ones((height, width)),
            K.zeros((height, width)),
        )
        return K.expand_dims(K.expand_dims(valid_seed_region, axis=0), axis=-1) 
Example 5
Project: group-ksparse-temporal-cnns   Author: srph25   File: ops.py    MIT License 6 votes vote down vote up
def diff(a, n=1, axis=-1):
    if axis == -1:
        axis = K.ndim(a) - 1
    a_aug = K.tile(K.expand_dims(a, axis=1), [1, n] + [1] * (K.ndim(a) - 1))
    pattern = (axis,) + tuple(set(range(K.ndim(a))) - {axis})
    inv_pattern = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, K.ndim(a)))
    def step(inputs, states):
        prev_output = states[0]
        t = states[1]
        t_int = K.cast(t[0], 'int32')
        prev_output_aug = K.permute_dimensions(prev_output, pattern)
        inputs_aug = K.permute_dimensions(inputs, pattern)
        output_aug = K.switch(K.all(t_int > 0),
                              K.concatenate([inputs_aug[:t_int], prev_output_aug[1:] - prev_output_aug[:-1]], axis=0),
                              K.concatenate([inputs_aug[:1], inputs_aug[1:] - inputs_aug[:-1]], axis=0))
        output = K.permute_dimensions(output_aug, inv_pattern)
        return output, [output, t + 1]
    d_aug = K.permute_dimensions(K.rnn(step, a_aug, [K.zeros_like(a), K.zeros((1,))])[0], pattern)
    d = K.permute_dimensions(d_aug[-(K.shape(a)[axis] - n):], inv_pattern)
    return d 
Example 6
Project: group-ksparse-temporal-cnns   Author: srph25   File: ops.py    MIT License 6 votes vote down vote up
def cumsum(a, n=1, axis=-1):
    if axis == -1:
        axis = K.ndim(a) - 1
    a_aug = K.tile(K.expand_dims(a, axis=1), [1, n] + [1] * (K.ndim(a) - 1))
    pattern = (axis,) + tuple(set(range(K.ndim(a))) - {axis})
    inv_pattern = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, K.ndim(a)))
    def step(inputs, states):
        prev_output = states[0]
        t = states[1]
        t_int = K.cast(t[0], 'int32')
        prev_output_aug = K.permute_dimensions(prev_output, pattern)
        inputs_aug = K.permute_dimensions(inputs, pattern)
        output_aug = K.switch(K.all(t_int > 0), K.cumsum(prev_output_aug, axis=0), K.cumsum(inputs_aug, axis=0))
        output = K.permute_dimensions(output_aug, inv_pattern)
        return output, [output, t + 1]
    c_aug = K.permute_dimensions(K.rnn(step, a_aug, [K.zeros_like(a), K.zeros((1,))])[0], pattern)
    c = K.permute_dimensions(c_aug[-(K.shape(a)[axis] - n):], inv_pattern)
    return c 
Example 7
Project: keras_bn_library   Author: bnsnapper   File: rnnrbm.py    MIT License 6 votes vote down vote up
def reset_states(self):
		assert self.stateful, 'Layer must be stateful.'
		input_shape = self.input_spec[0].shape

		if not input_shape[0]:
			raise Exception('If a RNN is stateful, a complete ' +
							'input_shape must be provided (including batch size).')

		if hasattr(self, 'states'):
			K.set_value(self.states[0],
			            np.zeros((input_shape[0], self.hidden_recurrent_dim)))
			K.set_value(self.states[1],
			            np.zeros((input_shape[0], self.input_dim)))
			K.set_value(self.states[2],
			            np.zeros((input_shape[0], self.hidden_dim)))
		else:
			self.states = [K.zeros((input_shape[0], self.hidden_recurrent_dim)),
							K.zeros((input_shape[0], self.input_dim)),
							K.zeros((input_shape[0], self.hidden_dim))] 
Example 8
Project: keras_bn_library   Author: bnsnapper   File: recurrent.py    MIT License 6 votes vote down vote up
def build(self, input_shape):
		self.input_spec = [InputSpec(shape=input_shape)]
		self.input_dim = input_shape[2]

		self.W = self.init((self.output_dim, 4 * self.input_dim),
		                   name='{}_W'.format(self.name))
		self.U = self.inner_init((self.input_dim, 4 * self.input_dim),
		                         name='{}_U'.format(self.name))
		self.b = K.variable(np.hstack((np.zeros(self.input_dim),
		                               K.get_value(self.forget_bias_init((self.input_dim,))),
		                               np.zeros(self.input_dim),
		                               np.zeros(self.input_dim))),
		                    name='{}_b'.format(self.name))

		self.A = self.init((self.input_dim, self.output_dim),
		                    name='{}_A'.format(self.name))
		self.ba = K.zeros((self.output_dim,), name='{}_ba'.format(self.name))


		self.trainable_weights = [self.W, self.U, self.b, self.A, self.ba]

		if self.initial_weights is not None:
			self.set_weights(self.initial_weights)
			del self.initial_weights 
Example 9
Project: keras_bn_library   Author: bnsnapper   File: recurrent.py    MIT License 6 votes vote down vote up
def reset_states(self):
		assert self.stateful, 'Layer must be stateful.'
		input_shape = self.input_spec[0].shape
		if not input_shape[0]:
			raise ValueError('If a RNN is stateful, it needs to know '
			                 'its batch size. Specify the batch size '
			                 'of your input tensors: \n'
			                 '- If using a Sequential model, '
			                 'specify the batch size by passing '
			                 'a `batch_input_shape` '
			                 'argument to your first layer.\n'
			                 '- If using the functional API, specify '
			                 'the time dimension by passing a '
			                 '`batch_shape` argument to your Input layer.')
		if hasattr(self, 'states'):
			K.set_value(self.states[0],
			            np.zeros((input_shape[0], self.input_dim)))
			K.set_value(self.states[1],
			            np.zeros((input_shape[0], self.output_dim)))
		else:
			self.states = [K.zeros((input_shape[0], self.input_dim)),
							K.zeros((input_shape[0], self.output_dim))] 
Example 10
Project: yoctol-keras-layer-zoo   Author: Yoctol   File: rnn_cell.py    GNU General Public License v3.0 6 votes vote down vote up
def reset_states(self, states=None):
        if states is None:
            self.recurrent_layer.reset_states(states)
        else:
            self.recurrent_layer.reset_states(states[:-1])

        batch_size = self.recurrent_layer.input_spec[0].shape[0]
        if self.dense_state is None:
            self.dense_state = K.zeros((
                batch_size,
                self.dense_layer.units
            ))
        elif states is None:
            K.set_value(
                self.dense_state,
                np.zeros((batch_size, self.dense_layer.units))
            )
        else:
            K.set_value(
                self.dense_state,
                states[-1]
            ) 
Example 11
Project: ChimeraNet   Author: leichtrhino   File: models.py    MIT License 6 votes vote down vote up
def loss_deepclustering(T, F, C, D):
    def _loss_deepclustering(y_true, y_pred):
        Y = K.reshape(y_true, (-1, T*F, C))
        V = K.reshape(y_pred, (-1, T*F, D))
        Dm = K.expand_dims(K.batch_dot(
            Y,
            K.sum(Y, axis=(1,)),
            axes=(2, 1)
        ))
        Dm = K.switch(Dm > 0, Dm**-0.25, K.zeros(K.shape(Dm)))
        DV, DY = Dm * V, Dm * Y
        a = K.sum(K.batch_dot(DV, DV, axes=(1, 1))**2, axis=(1, 2))
        b = K.sum(K.batch_dot(DV, DY, axes=(1, 1))**2, axis=(1, 2))
        c = K.sum(K.batch_dot(DY, DY, axes=(1, 1))**2, axis=(1, 2))
        return (a - 2*b + c)
    return _loss_deepclustering 
Example 12
Project: keras-lookahead   Author: CyberZHG   File: optimizers.py    MIT License 6 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = self.learning_rate * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            p_t = lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            self.updates.append(K.update_sub(p, p_t))
        return self.updates 
Example 13
Project: RPGOne   Author: RTHMaK   File: tree_composition_lstm.py    Apache License 2.0 6 votes vote down vote up
def get_initial_states(self, inputs):
        # The initial buffer is sent into the TreeLSTM as a part of the input.
        # i.e., inputs is a concatenation of the transitions and the initial buffer.
        # (batch_size, buffer_limit, output_dim+1)
        # We will now separate the buffer and the transitions and initialize the
        # buffer state of the TreeLSTM with the initial buffer value.
        # The rest of the input is the transitions, which we do not need now.

        # Take the buffer out.
        init_h_for_buffer = inputs[:, :, 1:]  # (batch_size, buffer_limit, output_dim)
        # Initializing all c as zeros.
        init_c_for_buffer = K.zeros_like(init_h_for_buffer)

        # Each element in the buffer is a concatenation of h and c for the corresponding
        # node
        init_buffer = K.concatenate([init_h_for_buffer, init_c_for_buffer], axis=-1)
        # We need a symbolic all zero tensor of size (samples, stack_limit, 2*output_dim) for
        # init_stack The problem is the first dim (samples) is a place holder and not an actual
        # value. So we'll use the following trick
        temp_state = K.zeros_like(inputs)  # (samples, buffer_ops_limit, input_dim)
        temp_state = K.tile(K.sum(temp_state, axis=(1, 2)),
                            (self.stack_limit, 2*self.output_dim, 1))  # (stack_limit, 2*output_dim, samples)
        init_stack = K.permute_dimensions(temp_state, (2, 0, 1))  # (samples, stack_limit, 2*output_dim)
        return [init_buffer, init_stack] 
Example 14
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 6 votes vote down vote up
def __init__(
            self,
            nb_domains, 
            domain_len, 
            init='glorot_uniform', 
            **kwargs):
        self.nb_domains = nb_domains
        self.domain_len = domain_len

        self.input = K.placeholder(ndim=4)
        
        self.init = lambda x: (
            (initializations.get(init)(x), None),
            (K.zeros((self.nb_domains,)), None) 
        )
        self.kwargs = kwargs
        
        self.W_shape = None
        self.W = None
        self.b = None
        
        super(ConvolutionCoOccupancy, self).__init__(**kwargs) 
Example 15
Project: onnx-keras   Author: leodestiny   File: custom_layers.py    MIT License 6 votes vote down vote up
def call(self, x, mask=None):
        b, ch, r, c = x.shape
        half_n = self.n // 2  # half the local region
        input_sqr = K.square(x)  # square the input

        extra_channels = K.zeros((b, int(ch) + 2 * half_n, r, c))
        input_sqr = K.concatenate(
            [extra_channels[:, :half_n, :, :], input_sqr, extra_channels[:, half_n + int(ch):, :, :]], axis=1)

        scale = self.k  # offset for the scale
        norm_alpha = self.alpha / self.n  # normalized alpha
        for i in range(self.n):
            scale += norm_alpha * input_sqr[:, i:i + int(ch), :, :]
        scale = scale ** self.beta
        x = x / scale
        return x 
Example 16
Project: MCLNN   Author: fadymedhat   File: layers.py    MIT License 6 votes vote down vote up
def construct_mask(self, feature_count, hidden_count, bandwidth, overlap, layer_is_masked):

        bw = bandwidth
        ov = overlap
        l = feature_count
        e = hidden_count

        a = np.arange(1, bw + 1)
        g = np.arange(1, int(np.ceil((l * e) / (l + bw - ov))) + 1)

        if layer_is_masked is False:
            binary_mask = np.ones([l, e])
        else:
            mask = np.zeros([l, e])
            flat_matrix = mask.flatten('F')

            for i in range(len(a)):
                for j in range(len(g)):
                    lx = a[i] + (g[j] - 1) * (l + bw - ov)
                    if lx <= l * e:
                        flat_matrix[lx - 1] = 1

            binary_mask =  np.transpose(flat_matrix.reshape(e, l))

        return binary_mask.astype(np.float32) 
Example 17
Project: dynamic_memory_networks_with_keras   Author: vchudinov   File: attention_cells.py    GNU General Public License v3.0 6 votes vote down vote up
def get_initial_state(self, inputs):
        return K.zeros(shape= [self.batch_size, self.units])
        # build an all-zero tensor of shape (samples, output_dim)
        # What the hell am I doing here?
      #  print("--------------------")
       # initial_state = K.zeros_like(inputs)  # (samples, timesteps, input_dim)
      #  print(initial_state.shape)
      #  initial_state = initial_state[:, :, :-1]
        #initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
        #print(initial_state.shape)
        #initial_state = K.expand_dims(initial_state)  # (samples, 1)
        #print(initial_state.shape)
        #raise SystemExit
      #  if hasattr(self.state_size, '__len__'):
      #      return [K.tile(initial_state, [1, dim])
       #             for dim in self.state_size]
       # else:
        #    return [K.tile(initial_state, [1, self.state_size])] 
Example 18
Project: FireDetectionCNN   Author: dogusyuksel   File: Custom_layers.py    MIT License 6 votes vote down vote up
def get_output(self, train):
        X = self.get_input(train)
        b, ch, r, c = K.shape(X)
        half_n = self.n // 2
        input_sqr = K.square(X)

        extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
        input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
                                   input_sqr,
                                   extra_channels[:, half_n + ch:, :, :]],
                                  axis=1)
        scale = self.k

        for i in range(self.n):
            scale += self.alpha * input_sqr[:, i:i + ch, :, :]
        scale = scale ** self.beta

        return X / scale 
Example 19
Project: FireDetectionCNN   Author: dogusyuksel   File: Custom_layers.py    MIT License 6 votes vote down vote up
def get_output(self, train):
        X = self.get_input(train)
        b, ch, r, c = K.shape(X)
        half_n = self.n // 2
        input_sqr = K.square(X)

        extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
        input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
                                   input_sqr,
                                   extra_channels[:, half_n + ch:, :, :]],
                                  axis=1)
        scale = self.k

        for i in range(self.n):
            scale += self.alpha * input_sqr[:, i:i + ch, :, :]
        scale = scale ** self.beta

        return X / scale 
Example 20
Project: VisualNN   Author: angelhunt   File: lrn.py    GNU General Public License v3.0 5 votes vote down vote up
def get_output(self, train):
        X = self.get_input(train)
        b, ch, r, c = K.shape(X)
        half_n = self.n // 2
        input_sqr = K.square(X)
        extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
        input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
                                   input_sqr,
                                   extra_channels[:, half_n + ch:, :, :]],
                                  axis=1)
        scale = self.k
        for i in range(self.n):
            scale += self.alpha * input_sqr[:, i:i + ch, :, :]
        scale = scale ** self.beta
        return X / scale 
Example 21
Project: deep-models   Author: LaurentMazare   File: rhn.py    Apache License 2.0 5 votes vote down vote up
def reset_states(self):
    assert self.stateful, 'Layer must be stateful.'
    input_shape = self.input_spec[0].shape
    if not input_shape[0]:
      raise Exception('If a RNN is stateful, a complete ' +
                      'input_shape must be provided (including batch size).')
    if hasattr(self, 'states'):
      K.set_value(self.states[0],
                  np.zeros((input_shape[0], self.output_dim)))
    else:
      self.states = [K.zeros((input_shape[0], self.output_dim))] 
Example 22
Project: deep-models   Author: LaurentMazare   File: lstm_ln.py    Apache License 2.0 5 votes vote down vote up
def build(self, input_shape):
    super(LSTM_LN, self).build(input_shape)
    self.gs, self.bs = [], []
    for i in xrange(3):
      f = 1 if i == 2 else 4
      self.gs += [ K.ones((f*self.output_dim,), name='{}_g%i'.format(self.name, i)) ]
      self.bs += [ K.zeros((f*self.output_dim,), name='{}_b%d'.format(self.name, i)) ]
    self.trainable_weights += self.gs + self.bs 
Example 23
Project: DeepLearningMugenKnock   Author: yoyoyo-yo   File: cgan_cifar10_keras.py    MIT License 5 votes vote down vote up
def test():
    # load trained model
    g, _ = G_model()
    g.load_weights('cgan_cifar10_keras.h5', by_name=True)

    np.random.seed(100)
    
    labels = ["air¥nplane", "auto¥bmobile", "bird", "cat", "deer",
              "dog", "frog", "horse", "ship", "truck"]
    
    con_x = np.zeros([10, num_classes])
    con_x[np.arange(10), np.arange(num_classes)] = 1
    
    for i in range(3):
        input_noise = np.random.uniform(-1, 1, size=(10, 100))
        g_output = g.predict(x={"x":input_noise, "con_x": con_x}, verbose=0)
        g_output = (g_output + 1 ) / 2

        for i in range(10):
            gen = g_output[i]

            if channel == 1:
                gen = gen[..., 0]
                cmap = "gray"
            elif channel == 3:
                cmap = None

            plt.subplot(1,10,i+1)
            plt.title(labels[i])
            plt.imshow(gen, cmap=cmap)
            plt.axis('off')

        plt.show() 
Example 24
Project: DeepLearningMugenKnock   Author: yoyoyo-yo   File: cgan_mnist_keras.py    MIT License 5 votes vote down vote up
def test():
    # load trained model
    g, _ = G_model()
    g.load_weights('cgan_cifar10_keras.h5', by_name=True)

    np.random.seed(100)
    
    con_x = np.zeros([10, num_classes])
    con_x[np.arange(10), np.arange(num_classes)] = 1
    
    for i in range(3):
        input_noise = np.random.uniform(-1, 1, size=(10, 100))
        g_output = g.predict(x={"x":input_noise, "con_x": con_x}, verbose=0)
        g_output = (g_output + 1 ) / 2

        for i in range(10):
            gen = g_output[i]
            
            if channel == 1:
                gen = gen[..., 0]
                cmap = "gray"
            elif channel == 3:
                cmap = None
                
            plt.subplot(1,10,i+1)
            plt.title(str(i))
            plt.imshow(gen, cmap=cmap)
            plt.axis('off')

        plt.show() 
Example 25
Project: fancy-cnn   Author: textclf   File: convolutions.py    MIT License 5 votes vote down vote up
def build(self):
        stack_size = self.input_shape[2]
        dtensor5 = T.TensorType('float32', (False,)*5)
        self.input = dtensor5()
        self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
        self.W = self.init(self.W_shape)
        self.b = shared_zeros((self.nb_filter,))

        self.params = [self.W, self.b]

        self.regularizers = []

        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights 
Example 26
Project: keras-attention-augmented-convs   Author: titu1994   File: attn_augconv.py    MIT License 5 votes vote down vote up
def rel_to_abs(self, x):
        shape = K.shape(x)
        shape = [shape[i] for i in range(3)]
        B, Nh, L, = shape
        col_pad = K.zeros(K.stack([B, Nh, L, 1]))
        x = K.concatenate([x, col_pad], axis=3)
        flat_x = K.reshape(x, [B, Nh, L * 2 * L])
        flat_pad = K.zeros(K.stack([B, Nh, L - 1]))
        flat_x_padded = K.concatenate([flat_x, flat_pad], axis=2)
        final_x = K.reshape(flat_x_padded, [B, Nh, L + 1, 2 * L - 1])
        final_x = final_x[:, :, :L, L - 1:]
        return final_x 
Example 27
Project: keras-lamb   Author: CyberZHG   File: optimizer.py    MIT License 5 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1
        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * t))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            mhat_t = m_t / (1. - K.pow(self.beta_1, t))
            vhat_t = v_t / (1. - K.pow(self.beta_2, t))

            u_t = mhat_t / K.sqrt(vhat_t + self.epsilon) + self.weight_decay * p
            trust_ratio = K.sqrt(K.sum(K.square(p)) / K.sum(K.square(u_t)))
            trust_ratio = K.minimum(K.maximum(trust_ratio, self.lower_bound), self.upper_bound)

            lr_p = trust_ratio * lr
            new_p = p - lr_p * u_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 28
Project: 360_aware_saliency   Author: MikhailStartsev   File: attentive_convlstm.py    GNU General Public License v3.0 5 votes vote down vote up
def get_initial_states(self, x):
        initial_state = K.sum(x, axis=1)
        initial_state = K.conv2d(initial_state, K.zeros((self.nb_filters_out, self.nb_filters_in, 1, 1)), border_mode='same')
        initial_states = [initial_state for _ in range(len(self.states))]

        return initial_states 
Example 29
Project: CapsNet-Fashion-MNIST   Author: subarnop   File: capsulelayers.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, num_capsule, dim_vector, num_routing=3,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 **kwargs):
        super(CapsuleLayer, self).__init__(**kwargs)
        self.num_capsule = num_capsule
        self.dim_vector = dim_vector
        self.num_routing = num_routing
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer) 
Example 30
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_v2.py    MIT License 5 votes vote down vote up
def _resource_apply_dense(self, grad, var):
        var_dtype = var.dtype.base_dtype
        lr_t = self._decayed_lr(var_dtype)
        momentum = array_ops.identity(self._get_hyper('momentum', var_dtype))
        if K_eval(momentum) != 0:
            m = self.get_slot(var, 'momentum')
        else:
            m = K.zeros(K.int_shape(var))
        total_iterations = self.total_iterations

        # Learning rate multipliers
        if self.lr_multipliers is not None:
            lr_t = _apply_lr_multiplier(self, lr_t, var)
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        v = momentum * m - self.eta_t*lr_t * grad  # velocity
        m = state_ops.assign(m, v, use_locking=self._use_locking)

        if self.nesterov:
            var_t = math_ops.sub(var, -momentum * v + self.eta_t*lr_t * grad)
        else:
            var_t = var + v

        # Weight decays
        if var.name in self.weight_decays.keys() and total_iterations != 0:
            var_t = _apply_weight_decays(self, var, var_t)

        iteration_done = self._updates_processed == (self._updates_per_iter - 1)
        _up = self._updates_processed
        self._updates_processed = (_up + 1) if not iteration_done else 0
        if iteration_done and not self._init_notified:
            self._init_notified = True

        var_update = state_ops.assign(var, var_t, use_locking=self._use_locking)
        t_cur = state_ops.assign_add(self.t_cur, int(iteration_done),
                                     use_locking=self._use_locking)
        updates = [var_update, m, t_cur]
        return control_flow_ops.group(*updates) 
Example 31
Project: gccaps   Author: tqbl   File: capsules.py    MIT License 5 votes vote down vote up
def __init__(self, n_capsules, dim_capsule, routings=3, use_bias=False,
                 kernel_initializer='glorot_uniform', bias_initializer='zeros',
                 **kwargs):
        super(CapsuleLayer, self).__init__(**kwargs)

        self.n_capsules = n_capsules
        self.dim_capsule = dim_capsule
        self.routings = routings
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer) 
Example 32
Project: gccaps   Author: tqbl   File: capsules.py    MIT License 5 votes vote down vote up
def call(self, inputs, training=None):
        """Apply transformation followed by capsule routing."""
        # Create dimension for output capsules and tile along this dim
        # (None, *n_capsules*, n_input_capsules, dim_input_capsules)
        inputs_tiled = K.tile(K.expand_dims(inputs, 1),
                              [1, self.n_capsules, 1, 1])

        # Apply linear transformation to compute prediction vectors
        inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]),
                              elems=inputs_tiled)
        # Add bias to prediction vectors if specified
        if self.use_bias:
            inputs_hat = K.bias_add(inputs_hat, self.bias,
                                    data_format='channels_first')

        # Initialize logit variables to zero
        b = K.zeros(shape=[K.shape(inputs_hat)[0],
                           self.n_capsules,
                           self.n_input_capsules])

        # Apply routing algorithm
        for i in range(self.routings):
            # Compute coupling coefficients
            c = K.softmax(b, axis=1)
            # Apple squashing function
            outputs = squash(K.batch_dot(c, inputs_hat, [2, 2]))
            # Update logits by computing agreement
            if i < self.routings - 1:
                b += K.batch_dot(outputs, inputs_hat, [2, 3])

        return outputs 
Example 33
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 5 votes vote down vote up
def build(self, input_shape):
        # input_dim = input_shape[2]
        # self.W_shape = (self.nb_filter, input_dim, self.filter_length, 1)
        # self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.b]
        # else:
        #     self.trainable_weights = [self.W]
        self.regularizers = []
        #
        # if self.W_regularizer:
        #     self.W_regularizer.set_param(self.W)
        #     self.regularizers.append(self.W_regularizer)
        #
        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)
        #
        # if self.activity_regularizer:
        #     self.activity_regularizer.set_layer(self)
        #     self.regularizers.append(self.activity_regularizer)
        #
        # self.constraints = {}
        # if self.W_constraint:
        #     self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint
        #
        # if self.initial_weights is not None:
        #     self.set_weights(self.initial_weights)
        #     del self.initial_weights 
Example 34
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 5 votes vote down vote up
def build(self, input_shape):
        if self.dim_ordering == 'th':
            stack_size = input_shape[1]
            self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
        elif self.dim_ordering == 'tf':
            stack_size = input_shape[3]
            self.W_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
        else:
            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        # self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
        if self.bias:
            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
            self.trainable_weights = [self.b]
        # else:
        #     self.trainable_weights = [self.W]
        self.regularizers = []

        # if self.W_regularizer:
        #     self.W_regularizer.set_param(self.W)
        #     self.regularizers.append(self.W_regularizer)

        if self.bias and self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        # if self.W_constraint:
        #     self.constraints[self.W] = self.W_constraint
        if self.bias and self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        # if self.initial_weights is not None:
        #     self.set_weights(self.initial_weights)
        #     del self.initial_weights 
Example 35
Project: keras_bn_library   Author: bnsnapper   File: recurrent.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
		self.input_spec = [InputSpec(shape=input_shape)]
		input_dim = input_shape[2]
		self.input_dim = input_dim
		
		if self.stateful:
			self.reset_states()
		else:
			self.states = [None, None]
			self.states_dim = [self.input_dim, self.output_dim]


		self.weight_size = self.output_dim * 4
		self.W = self.add_weight((input_dim, self.weight_size),
                                 initializer=self.init,
                                 name='{}_W'.format(self.name),
                                 regularizer=self.W_regularizer)
		self.U = self.add_weight((input_dim, self.weight_size),
                                 initializer=self.inner_init,
                                 name='{}_U'.format(self.name),
                                 regularizer=self.U_regularizer)

		def b_reg(shape, name=None):
			return K.variable(np.hstack((np.zeros(self.output_dim),
										K.get_value(self.forget_bias_init((self.output_dim,))),
										np.zeros(self.output_dim),
										np.zeros(self.output_dim))),
										name='{}_b'.format(self.name))
		self.b = self.add_weight((self.weight_size,),
                                     initializer=b_reg,
                                     name='{}_b'.format(self.name),
                                     regularizer=self.b_regularizer)


		if self.initial_weights is not None:
			self.set_weights(self.initial_weights)
			del self.initial_weights

		self.built = True 
Example 36
Project: deep-pmsm   Author: wkirgsn   File: custom_layers.py    MIT License 5 votes vote down vote up
def __init__(self,
                 center=True,
                 scale=True,
                 epsilon=None,
                 gamma_initializer='ones',
                 beta_initializer='zeros',
                 gamma_regularizer=None,
                 beta_regularizer=None,
                 gamma_constraint=None,
                 beta_constraint=None,
                 **kwargs):
        """Layer normalization layer
        See: [Layer Normalization](https://arxiv.org/pdf/1607.06450.pdf)
        :param center: If True, add offset of `beta` to normalized tensor. If False, `beta` is ignored.
        :param scale: If True, multiply by `gamma`. If False, `gamma` is not used.
        :param epsilon: Epsilon for variance.
        :param gamma_initializer: Initializer for the gamma weight.
        :param beta_initializer: Initializer for the beta weight.
        :param gamma_regularizer: Optional regularizer for the gamma weight.
        :param beta_regularizer: Optional regularizer for the beta weight.
        :param gamma_constraint: Optional constraint for the gamma weight.
        :param beta_constraint: Optional constraint for the beta weight.
        :param kwargs:
        """
        super(LayerNormalization, self).__init__(**kwargs)
        self.supports_masking = True
        self.center = center
        self.scale = scale
        if epsilon is None:
            epsilon = K.epsilon() * K.epsilon()
        self.epsilon = epsilon
        self.gamma_initializer = initializers.get(gamma_initializer)
        self.beta_initializer = initializers.get(beta_initializer)
        self.gamma_regularizer = regularizers.get(gamma_regularizer)
        self.beta_regularizer = regularizers.get(beta_regularizer)
        self.gamma_constraint = constraints.get(gamma_constraint)
        self.beta_constraint = constraints.get(beta_constraint)
        self.gamma, self.beta = None, None 
Example 37
Project: NTM-Keras   Author: SigmaQuan   File: lstm2ntm.py    MIT License 5 votes vote down vote up
def reset_states(self):
        assert self.stateful, 'Layer must be stateful.'
        input_shape = self.input_spec[0].shape
        if not input_shape[0]:
            raise Exception('If a RNN is stateful, a complete ' +
                            'input_shape must be provided (including batch size).')
        if hasattr(self, 'states'):
            K.set_value(self.states[0],
                        np.zeros((input_shape[0], self.output_dim)))
            K.set_value(self.states[1],
                        np.zeros((input_shape[0], self.output_dim)))
        else:
            self.states = [K.zeros((input_shape[0], self.output_dim)),
                           K.zeros((input_shape[0], self.output_dim))] 
Example 38
Project: NTM-Keras   Author: SigmaQuan   File: memory.py    MIT License 5 votes vote down vote up
def initial(number_of_memory_locations, memory_vector_size):
    return K.zeros((number_of_memory_locations, memory_vector_size)) 
Example 39
Project: keras-optimizers   Author: hi-im-ryanli   File: frankenstein.py    MIT License 5 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            v_t = K.maximum(self.beta_2 * v, K.abs(g))
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 40
Project: CapsNet-Collections   Author: Jiankai-Sun   File: capsulelayers.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, num_capsule, dim_vector, num_routing=3,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 **kwargs):
        super(CapsuleLayer, self).__init__(**kwargs)
        self.num_capsule = num_capsule
        self.dim_vector = dim_vector
        self.num_routing = num_routing
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer) 
Example 41
Project: conv_qsar_fast   Author: connorcoley   File: GraphEmbedding_sumAfter.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
		'''Builds internal weights and paramer attribute'''
		# NOTE: NEED TO TILE AND EVALUATE SO THAT PARAMS CAN BE VARIABLES
		# OTHERWISE K.GET_VALUE() DOES NOT WORK

		# Define template weights for inner FxF
		W_inner = self.init_inner((self.inner_dim, self.inner_dim))
		b_inner = K.zeros((1, self.inner_dim))
		# Initialize weights tensor 
		self.W_inner = K.variable(T.tile(W_inner, (self.depth + 1, 1, 1)).eval() + \
			initializations.uniform((self.depth + 1, self.inner_dim, self.inner_dim)).eval())
		self.W_inner.name = 'T:W_inner'
		self.b_inner = K.variable(T.tile(b_inner, (self.depth + 1, 1, 1)).eval()  + \
			initializations.uniform((self.depth + 1, 1, self.inner_dim)).eval())
		self.b_inner.name = 'T:b_inner'
		# # Concatenate third dimension (depth) so different layers can have 
		# # different weights. Now, self.W_inner[#,:,:] corresponds to the 
		# # weight matrix for layer/depth #.

		# Define template weights for output FxL
		W_output = self.init_output((self.inner_dim, self.output_dim), scale = self.scale_output)
		b_output = K.zeros((1, self.output_dim))
		# Initialize weights tensor
		self.W_output = K.variable(T.tile(W_output, (self.depth + 1, 1, 1)).eval())
		self.W_output.name = 'T:W_output'
		self.b_output = K.variable(T.tile(b_output, (self.depth + 1, 1, 1)).eval())
		self.b_output.name = 'T:b_output'
		# # Concatenate third dimension (depth) so different layers can have 
		# # different weights. Now, self.W_output[#,:,:] corresponds to the 
		# # weight matrix for layer/depth #.

		# Pack params
		self.trainable_weights = [self.W_inner, 
					   self.b_inner,
					   self.W_output,
					   self.b_output]
		self.params = [self.W_inner, 
					   self.b_inner,
					   self.W_output,
					   self.b_output] 
Example 42
Project: conv_qsar_fast   Author: connorcoley   File: GraphEmbedding.py    MIT License 5 votes vote down vote up
def build(self, input_shape):
		'''Builds internal weights and paramer attribute'''
		# NOTE: NEED TO TILE AND EVALUATE SO THAT PARAMS CAN BE VARIABLES
		# OTHERWISE K.GET_VALUE() DOES NOT WORK

		# Define template weights for inner FxF
		W_inner = self.init_inner((self.inner_dim, self.inner_dim))
		b_inner = K.zeros((1, self.inner_dim))
		# Initialize weights tensor 
		self.W_inner = K.variable(T.tile(W_inner, (self.depth + 1, 1, 1)).eval() + \
			initializations.uniform((self.depth + 1, self.inner_dim, self.inner_dim)).eval())
		self.W_inner.name = 'T:W_inner'
		self.b_inner = K.variable(T.tile(b_inner, (self.depth + 1, 1, 1)).eval()  + \
			initializations.uniform((self.depth + 1, 1, self.inner_dim)).eval())
		self.b_inner.name = 'T:b_inner'
		# # Concatenate third dimension (depth) so different layers can have 
		# # different weights. Now, self.W_inner[#,:,:] corresponds to the 
		# # weight matrix for layer/depth #.

		# Define template weights for output FxL
		W_output = self.init_output((self.inner_dim, self.output_dim), scale = self.scale_output)
		b_output = K.zeros((1, self.output_dim))
		# Initialize weights tensor
		self.W_output = K.variable(T.tile(W_output, (self.depth + 1, 1, 1)).eval())
		self.W_output.name = 'T:W_output'
		self.b_output = K.variable(T.tile(b_output, (self.depth + 1, 1, 1)).eval())
		self.b_output.name = 'T:b_output'
		# # Concatenate third dimension (depth) so different layers can have 
		# # different weights. Now, self.W_output[#,:,:] corresponds to the 
		# # weight matrix for layer/depth #.

		# Pack params
		self.trainable_weights = [self.W_inner, 
					   self.b_inner,
					   self.W_output,
					   self.b_output]
		self.params = [self.W_inner, 
					   self.b_inner,
					   self.W_output,
					   self.b_output] 
Example 43
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(
            self, nb_motifs, motif_len, 
            init='glorot_uniform', 
            **kwargs):
        self.nb_motifs = nb_motifs
        self.motif_len = motif_len
        self.input = K.placeholder(ndim=4)
        
        self.init = lambda x: (
            initializations.get(init)(x), 
            K.zeros((self.nb_motifs,)) 
        )
        super(ConvolutionDNAShapeBinding, self).__init__(**kwargs) 
Example 44
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(
            self,
            nb_motifs,
            motif_len, 
            use_three_base_encoding=True,
            init='glorot_uniform', 
            **kwargs):
        self.nb_motifs = nb_motifs
        self.motif_len = motif_len
        self.input = K.placeholder(ndim=4)
        self.use_three_base_encoding = use_three_base_encoding
        self.kwargs = kwargs
        
        self.W = None
        self.b = None
        
        #if isinstance(init, ConvolutionalDNABindingModel):
        #    self.init = lambda x: (
        #        K.variable(-init.ddg_array[None,None,:,:]), 
        #        K.variable(np.array([-init.ref_energy,])[:,None]) 
        #    )
        #else:
        #    self.init = lambda x: (
        #        initializations.get(init)(x), 
        #        K.zeros((self.nb_motifs,)) 
        #    )
        self.init = initializations.get(init)
        super(ConvolutionDNASequenceBinding, self).__init__(**kwargs) 
Example 45
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 5 votes vote down vote up
def init_filters(self):
        if self.use_three_base_encoding:
            self.W_shape = (
                self.nb_motifs, 3, 1, self.motif_len)
        else:
            self.W_shape = (
                self.nb_motifs, 4, 1, self.motif_len)
        self.W = self.init(self.W_shape) 
        self.b = K.zeros((self.nb_motifs,))
        return 
Example 46
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 5 votes vote down vote up
def extract_binding_models(self):
        mos = []
        for i in xrange(self.nb_motifs):
            ddg_array = self.W[0].get_value()[i,:,0,:]
            ddg_array = np.vstack(
                (np.zeros((1, ddg_array.shape[1])), ddg_array)).T
            ref_energy = self.b[0].get_value()[i]
            mos.append(EnergeticDNABindingModel(-ref_energy, -ddg_array))
        return mos 
Example 47
Project: onnx-keras   Author: leodestiny   File: backend.py    MIT License 5 votes vote down vote up
def _pad(cls, x, pad, dim):
        import copy
        for i in range(dim):
            if pad[i] == 0 and pad[i + dim] == 0:
                continue
            shape = K.int_shape(x)
            zero_shape = copy.deepcopy(list(shape))
            zero_shape[i] = pad[i]
            zeros1 = K.zeros(zero_shape)
            zero_shape[i] = pad[i + dim]
            zeros2 = K.zeros(zero_shape)
            x = K.concatenate([zeros1, x, zeros2], axis=i)
        return x 
Example 48
Project: stochastic_depth_keras   Author: dblN   File: train.py    MIT License 5 votes vote down vote up
def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
    global add_tables

    nb_filter = output_shape[0]
    conv = Convolution2D(nb_filter, 3, 3, subsample=strides,
                         border_mode="same", W_regularizer=l2(weight_decay))(x)
    conv = BatchNormalization(axis=1)(conv)
    conv = Activation("relu")(conv)
    conv = Convolution2D(nb_filter, 3, 3,
                         border_mode="same", W_regularizer=l2(weight_decay))(conv)
    conv = BatchNormalization(axis=1)(conv)

    if strides[0] >= 2:
        x = AveragePooling2D(strides)(x)

    if (output_shape[0] - input_shape[0]) > 0:
        pad_shape = (1,
                     output_shape[0] - input_shape[0],
                     output_shape[1],
                     output_shape[2])
        padding = K.zeros(pad_shape)
        padding = K.repeat_elements(padding, K.shape(x)[0], axis=0)
        x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
                   output_shape=output_shape)(x)

    _death_rate = K.variable(death_rate)
    scale = K.ones_like(conv) - _death_rate
    conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
                  output_shape=output_shape)(conv)

    out = merge([conv, x], mode="sum")
    out = Activation("relu")(out)

    gate = K.variable(1, dtype="uint8")
    add_tables += [{"death_rate": _death_rate, "gate": gate}]
    return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
                  output_shape=output_shape)([out, x]) 
Example 49
Project: CDSGD   Author: SCSLabISU   File: FASGD.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def update_mean_parameters(agentmodels,n_agents):
    parameters=[0 for nb in range(n_agents)]
    for nb in range(n_agents):
        parameters[nb]=agentmodels[nb]._collected_trainable_weights

    info_shapes = [K.get_variable_shape(p) for p in agentmodels[0]._collected_trainable_weights]
    parameter_mean = [K.zeros(info_shape) for info_shape in info_shapes]

    for i in range(len(parameter_mean)):
        for nb in range(n_agents):
            parameter_mean[i]+=(1/n_agents)*parameters[nb][i]

    for nb in range(n_agents):
        agentmodels[nb]._collected_trainable_weights=parameter_mean
    return agentmodels 
Example 50
Project: dynamic_memory_networks_with_keras   Author: vchudinov   File: attention_cells.py    GNU General Public License v3.0 5 votes vote down vote up
def call(self, input_list, initial_state=None, mask=None, training=None):

        inputs = input_list

        self._generate_dropout_mask(inputs, training=training)
        self._generate_recurrent_dropout_mask(inputs, training=training)

        # if has_arg(self.layer.call, 'training'):
        self.training = training
        uses_learning_phase = False
      #  initial_state = self.get_initial_state(inputs)
        last_output, outputs, _ = K.rnn(self.step,
                                        inputs=inputs,
                                        constants=[],
                                        initial_states=[K.zeros(shape= [self.batch_size, self.units])],
                                        #input_length=input_shape,
                                        unroll=False)
        if self.return_sequences:
            y = outputs
        else:
            y = last_output

        if (hasattr(self, 'activity_regularizer') and
                self.activity_regularizer is not None):
            regularization_loss = self.activity_regularizer(y)
            self.add_loss(regularization_loss, inputs)

        if uses_learning_phase:
            y._uses_learning_phase = True

        if self.return_sequences:
            timesteps = input_shape[1]
            new_time_steps = list(y.get_shape())
            new_time_steps[1] = timesteps
            y.set_shape(new_time_steps)
        return y 
Example 51
Project: Keras-TextClassification   Author: yongzhuo   File: keras_radam.py    MIT License 5 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        beta_1_t = K.pow(self.beta_1, t)
        beta_2_t = K.pow(self.beta_2, t)
        rho = 2 / (1 - self.beta_2) - 1
        rho_t = rho - 2 * t * beta_2_t / (1 - beta_2_t)
        r_t = K.sqrt(
            K.relu(rho_t - 4) * K.relu(rho_t - 2) * rho / ((rho - 4) * (rho - 2) * rho_t)
        )
        flag = K.cast(rho_t > 4, K.floatx())

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            mhat_t = m_t / (1 - beta_1_t)
            vhat_t = K.sqrt(v_t / (1 - beta_2_t))
            p_t = p - lr * mhat_t * (flag * r_t / (vhat_t + self.epsilon) + (1 - flag))

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 52
Project: Implementation-CVPR2015-CNN-for-ReID   Author: Ning-Ding   File: model_for_market1501.py    MIT License 5 votes vote down vote up
def next(self):
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        batch_x1 = np.zeros(tuple([current_batch_size * 2] + [128,64,3]))
        batch_x2 = np.zeros(tuple([current_batch_size * 2] + [128,64,3]))
        batch_y  = np.zeros([current_batch_size * 2, 2])
        for i, j in enumerate(index_array):
            x1 = np.array(Image.open(self.folder_dir + self.f[self.train_or_validation][j,0])) / 255.
            x2 = np.array(Image.open(self.folder_dir + self.f[self.train_or_validation][j,1])) / 255.  
            if np.random.rand() > self.flag:
                x1 = self.image_data_generator.random_transform(x1.astype('float32'))
            if np.random.rand() > self.flag:
                x2 = self.image_data_generator.random_transform(x2.astype('float32'))
            batch_x1[2*i] = x1
            batch_x2[2*i] = x2
            batch_y[2*i][1] = 1
            while True:
                index_1,index_2 = np.random.choice(self.path_list,2)
                if index_1[6] != index_2[6] and index_1[0:4] != index_2[0:4]:
                    break
            x1 = np.array(Image.open(self.folder_dir + index_1)) / 255.
            x2 = np.array(Image.open(self.folder_dir + index_2)) / 255.
            batch_x1[2*i+1] = x1
            batch_x2[2*i+1] = x2
            batch_y[2*i+1][0] = 1
            
        return [batch_x1,batch_x2], batch_y 
Example 53
Project: deep-models   Author: LaurentMazare   File: rhn.py    Apache License 2.0 4 votes vote down vote up
def build(self, input_shape):
    self.input_spec = [InputSpec(shape=input_shape)]
    self.input_dim = input_shape[2]

    if self.stateful:
      self.reset_states()
    else:
      # initial states: all-zero tensor of shape (output_dim)
      self.states = [None]

    self.W_t = self.init((self.input_dim, self.output_dim),
                         name='{}_W_t'.format(self.name))
    self.b_t = K.zeros((self.output_dim,), name='{}_b_t'.format(self.name))
    self.W_h = self.init((self.input_dim, self.output_dim),
                         name='{}_W_h'.format(self.name))
    self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))

    self.U_ts, self.b_ts = [], []
    self.U_hs, self.b_hs = [], []
    for l in xrange(self.L):
      self.U_ts.append(self.inner_init((self.output_dim, self.output_dim), name='{}_U_t{}'.format(self.name, l)))
      self.b_ts.append(K.zeros((self.output_dim,), name='{}_b_t{}'.format(self.name, l)))
      self.U_hs.append(self.inner_init((self.output_dim, self.output_dim), name='{}_U_h{}'.format(self.name, l)))
      self.b_hs.append(K.zeros((self.output_dim,), name='{}_b_h{}'.format(self.name, l)))

    self.trainable_weights = [ self.W_t, self.b_t, self.W_h, self.b_h] + self.U_ts + self.U_hs + self.b_ts + self.b_hs

    self.W = K.concatenate([self.W_t, self.W_h])
    self.U = K.concatenate(self.U_ts + self.U_hs)
    self.b = K.concatenate([self.b_t, self.b_h] + self.b_ts + self.b_hs)

    self.regularizers = []
    if self.W_regularizer:
      self.W_regularizer.set_param(self.W)
      self.regularizers.append(self.W_regularizer)
    if self.U_regularizer:
      self.U_regularizer.set_param(self.U)
      self.regularizers.append(self.U_regularizer)
    if self.b_regularizer:
      self.b_regularizer.set_param(self.b)
      self.regularizers.append(self.b_regularizer)

    if self.initial_weights is not None:
      self.set_weights(self.initial_weights)
      del self.initial_weights 
Example 54
Project: DeepLearningMugenKnock   Author: yoyoyo-yo   File: cgan_cifar10_keras.py    MIT License 4 votes vote down vote up
def G_model():
    inputs = Input([100, ], name="x")
    con_x = Input([num_classes, ], name="con_x")
    con_x2 = Input([img_height, img_width, num_classes], name="con_x2")
    
    #con_x = K.zeros([None, num_classes, 1, 1])
    #print(con_x.shape)
    #con_x = np.zeros([len(_con_x), num_classes, 1, 1], dtype=np.float32)
    #con_x[np.arange(len(_con_x)), _con_x] = 1

    x = concatenate([inputs, con_x], axis=-1)
    
    in_h = int(img_height / 16)
    in_w = int(img_width / 16)
    d_dim = 256
    base = 128
    x = Dense(in_h * in_w * d_dim, name='g_dense1',
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Reshape((in_h, in_w, d_dim), input_shape=(d_dim * in_h * in_w,))(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_dense1_bn')(x)
    # 1/8
    x = Conv2DTranspose(base*4, (5, 5), name='g_conv1', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv1_bn')(x)
    # 1/4
    x = Conv2DTranspose(base*2, (5, 5), name='g_conv2', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv2_bn')(x)
    # 1/2
    x = Conv2DTranspose(base, (5, 5), name='g_conv3', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv3_bn')(x)
    # 1/1
    x = Conv2DTranspose(channel, (5, 5), name='g_out', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02),  bias_initializer=Constant())(x)
    x = Activation('tanh')(x)

    #con_x = np.zerns([len(_con_x), num_classes, img_height, img_width], dtype=np.float32)
    #con_x[np.arange(len(_con_x)), _con_x] = 1
    x2 = concatenate([x, con_x2], axis=-1)

    model = Model(inputs=[inputs, con_x, con_x2], outputs=[x], name='G')
    gan_g_model = Model(inputs=[inputs, con_x, con_x2], outputs=[x2], name='GAN_G')
    
    return model, gan_g_model 
Example 55
Project: DeepLearningMugenKnock   Author: yoyoyo-yo   File: cgan_mnist_keras.py    MIT License 4 votes vote down vote up
def G_model():
    inputs = Input([100, ], name="x")
    con_x = Input([num_classes, ], name="con_x")
    con_x2 = Input([img_height, img_width, num_classes], name="con_x2")
    
    #con_x = K.zeros([None, num_classes, 1, 1])
    #print(con_x.shape)
    #con_x = np.zeros([len(_con_x), num_classes, 1, 1], dtype=np.float32)
    #con_x[np.arange(len(_con_x)), _con_x] = 1

    x = concatenate([inputs, con_x], axis=-1)
    
    in_h = int(img_height / 4)
    in_w = int(img_width / 4)
    d_dim = 256
    base = 128
    x = Dense(in_h * in_w * d_dim, name='g_dense1',
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Reshape((in_h, in_w, d_dim), input_shape=(d_dim * in_h * in_w,))(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_dense1_bn')(x)
    # 1/8
    #x = Conv2DTranspose(base*4, (5, 5), name='g_conv1', padding='same', strides=(2,2),
    #    kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    #x = Activation('relu')(x)
    #x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv1_bn')(x)
    # 1/4
    #x = Conv2DTranspose(base*2, (5, 5), name='g_conv2', padding='same', strides=(2,2),
    #    kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    #x = Activation('relu')(x)
    #x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv2_bn')(x)
    # 1/2
    x = Conv2DTranspose(base, (5, 5), name='g_conv3', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
    x = Activation('relu')(x)
    x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv3_bn')(x)
    # 1/1
    x = Conv2DTranspose(channel, (5, 5), name='g_out', padding='same', strides=(2,2),
        kernel_initializer=RN(mean=0.0, stddev=0.02),  bias_initializer=Constant())(x)
    x = Activation('tanh')(x)

    #con_x = np.zerns([len(_con_x), num_classes, img_height, img_width], dtype=np.float32)
    #con_x[np.arange(len(_con_x)), _con_x] = 1
    x2 = concatenate([x, con_x2], axis=-1)

    model = Model(inputs=[inputs, con_x], outputs=[x], name='G')
    gan_g_model = Model(inputs=[inputs, con_x, con_x2], outputs=[x2], name='GAN_G')
    
    return model, gan_g_model 
Example 56
Project: bert_lamb_pretrain   Author: goldenbili   File: keras_lamb.py    Apache License 2.0 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            m_t_hat = m_t / (1. - K.pow(self.beta_1, t))
            v_t_hat = v_t / (1. - K.pow(self.beta_2, t))

            p_dash = m_t_hat / (K.sqrt(v_t_hat + self.epsilon))

            if self.weight_decay > 0.:
                wd = self.weight_decay * p
                p_dash = p_dash + wd

            r1 = K.sqrt(K.sum(K.square(p)))
            r2 = K.sqrt(K.sum(K.square(p_dash)))

            r = tf.where(tf.greater(r1, 0.),
                         tf.where(tf.greater(r2, 0.),
                                  r1 / r2,
                                  1.0),
                         1.0)
            # r = r1 / r2
            eta = r * lr

            p_t = p - eta * p_dash

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 57
Project: keras_radam   Author: forin-xyz   File: radam.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay:
            lr = lr * (1. / (1. + self.decay * K.cast(
                self.iterations, K.dtype(self.decay)
            )))

        t = K.cast(self.iterations, K.floatx()) + 1.
        beta_1 = self.beta_1
        beta_2 = self.beta_2
        beta_1_t = K.pow(beta_1, t)
        beta_2_t = K.pow(beta_2, t)
        rho_inf = 2. / (1. - beta_2) - 1.
        rho_t = rho_inf - 2. * t * beta_2_t / (1. - beta_2_t)
        r_t = K.sqrt(
            K.relu(rho_t - 4.) * (rho_t - 2.) * rho_inf / (
                (rho_inf - 4.) * (rho_inf - 2.) * rho_t )
        )
        flag = K.cast(rho_t > 4., K.floatx())

        ms = [K.zeros(K.int_shape(p)) for p in params]
        vs = [K.zeros(K.int_shape(p)) for p in params]

        self.weights = [self.iterations] + ms + vs
        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = beta_1 * m + (1. - beta_1) * g
            v_t = beta_2 * v + (1. - beta_2) * K.square(g)

            m_hat_t = m_t / (1. - beta_1_t)
            v_hat_t = K.sqrt(v_t / (1. - beta_2_t))
            new_p = p - lr * (r_t / (v_hat_t + self.epsilon) + flag - 1.)* m_hat_t

            if getattr(p, "constraint", None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
        return self.updates 
Example 58
Project: CapsNet-Fashion-MNIST   Author: subarnop   File: capsulelayers.py    GNU General Public License v3.0 4 votes vote down vote up
def call(self, inputs, training=None):
        # inputs.shape=[None, input_num_capsule, input_dim_vector]
        # Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector]
        inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2)

        # Replicate num_capsule dimension to prepare being multiplied by W
        # Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector]
        inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1])

        """
        # Begin: inputs_hat computation V1 ---------------------------------------------------------------------#
        # Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size.
        # w_tiled.shape = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector]
        w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1])

        # Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
        inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3])
        # End: inputs_hat computation V1 ---------------------------------------------------------------------#
        """

        # Begin: inputs_hat computation V2 ---------------------------------------------------------------------#
        # Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow.
        # inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
        inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]),
                             elems=inputs_tiled,
                             initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector]))
        # End: inputs_hat computation V2 ---------------------------------------------------------------------#
        """
        # Begin: routing algorithm V1, dynamic ------------------------------------------------------------#
        def body(i, b, outputs):
            c = tf.nn.softmax(b, dim=2)  # dim=2 is the num_capsule dimension
            outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
            if i != 1:
                b = b + K.sum(inputs_hat * outputs, -1, keepdims=True)
            return [i-1, b, outputs]
        cond = lambda i, b, inputs_hat: i > 0
        loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)]
        shape_invariants = [tf.TensorShape([]),
                            tf.TensorShape([None, self.input_num_capsule, self.num_capsule, 1, 1]),
                            tf.TensorShape([None, 1, self.num_capsule, 1, self.dim_vector])]
        _, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants)
        # End: routing algorithm V1, dynamic ------------------------------------------------------------#
        """

        # Begin: routing algorithm V2, static -----------------------------------------------------------#
        # Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance
        assert self.num_routing > 0, 'The num_routing should be > 0.'
        for i in range(self.num_routing):
            c = tf.nn.softmax(self.bias, dim=2)  # dim=2 is the num_capsule dimension
            # outputs.shape=[None, 1, num_capsule, 1, dim_vector]
            outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))

            # last iteration needs not compute bias which will not be passed to the graph any more anyway.
            if i != self.num_routing - 1:
                # self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True))
                self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True)
            # tf.summary.histogram('BigBee', self.bias)  # for debugging
        # End: routing algorithm V2, static ------------------------------------------------------------#

        return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector]) 
Example 59
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        lr = self.learning_rate
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))
        # momentum
        shapes = [K.int_shape(p) for p in params]
        moments = [K.zeros(shape, name='moment_' + str(i))
                   for (i, shape) in enumerate(shapes)]
        self.weights = [self.iterations] + moments

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = lr * self.eta_t  # for external tracking

        for p, g, m in zip(params, grads, moments):
            # Learning rate multipliers
            lr_t = self.learning_rate
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t, p)

            v = self.momentum * m - self.eta_t * lr_t * g  # velocity
            self.updates.append(K.update(m, v))

            if self.nesterov:
                p_t = p + self.momentum * v - self.eta_t * lr_t * g
            else:
                p_t = p + v

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 60
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]

        if self.amsgrad:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        else:
            vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = lr_t * self.eta_t  # for external tracking
        lr_t_premult = lr_t

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            # Learning rate multipliers
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t_premult, p)

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            if self.amsgrad:
                vhat_t = K.maximum(vhat, v_t)
                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, vhat_t))
            else:
                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 61
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        t = K.cast(self.iterations, K.floatx()) + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (
            K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = self.lr * self.eta_t  # for external tracking

        for p, g, m, v in zip(params, grads, ms, vs):
            # Learning rate multipliers
            lr_t = self.lr
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t, p)

            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + (
                momentum_cache_t_1 * m_t_prime)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            p_t = p - self.eta_t * lr_t * m_t_bar / (
                    K.sqrt(v_t_prime) + self.epsilon)

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 62
Project: keras-adamw   Author: OverLordGoldDragon   File: optimizers_225.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]
        self.updates.append(K.update_add(self.t_cur, 1))

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))
        # momentum
        shapes = [K.int_shape(p) for p in params]
        moments = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + moments

        total_iterations = self.total_iterations
        # Cosine annealing
        if self.use_cosine_annealing and total_iterations != 0:
            self.eta_t = _compute_eta_t(self)
        self.lr_t = lr * self.eta_t  # for external tracking

        for p, g, m in zip(params, grads, moments):
            # Learning rate multipliers
            lr_t = self.lr
            if self.lr_multipliers is not None:
                lr_t = _apply_lr_multiplier(self, lr_t, p)

            v = self.momentum * m - self.eta_t * lr_t * g  # velocity
            self.updates.append(K.update(m, v))

            if self.nesterov:
                p_t = p + self.momentum * v - self.eta_t * lr_t * g
            else:
                p_t = p + v

            # Weight decays
            if p.name in self.weight_decays.keys() and total_iterations != 0:
                p_t = _apply_weight_decays(self, p, p_t)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))

        self._init_notified = True
        return self.updates 
Example 63
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 4 votes vote down vote up
def call(self, x, mask=None):
        # input shape: (nb_samples, time (padded with zeros), input_dim)
        input_shape = self.input_spec[0].shape
        if K._BACKEND == 'tensorflow':
            if not input_shape[1]:
                raise Exception('When using TensorFlow, you should define '
                                'explicitly the number of timesteps of '
                                'your sequences.\n'
                                'If your first layer is an Embedding, '
                                'make sure to pass it an "input_length" '
                                'argument. Otherwise, make sure '
                                'the first layer has '
                                'an "input_shape" or "batch_input_shape" '
                                'argument, including the time axis. '
                                'Found input shape at layer ' + self.name +
                                ': ' + str(input_shape))

        if self.layer.stateful:
            initial_states = self.layer.states
        else:
            initial_states = self.layer.get_initial_states(x)
        constants = self.get_constants(x)
        preprocessed_input = self.layer.preprocess_input(x)
        

        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
                                             initial_states,
                                             go_backwards=self.layer.go_backwards,
                                             mask=mask,
                                             constants=constants,
                                             unroll=self.layer.unroll,
                                             input_length=input_shape[1])
        if self.layer.stateful:
            self.updates = []
            for i in range(len(states)):
                self.updates.append((self.layer.states[i], states[i]))

        if self.layer.return_sequences:
            return outputs
        else:
            return last_output
            
            
# test likes in https://github.com/fchollet/keras/blob/master/tests/keras/layers/test_wrappers.py 
Example 64
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 65
Project: keras_extension   Author: k1414st   File: optimizers.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            eta_l_t = self.terminal_bound - \
                (self.terminal_bound - self.lower_bound) / \
                ((1. - self.beta_2) * t + 1)
            eta_u_t = self.terminal_bound + \
                (self.upper_bound - self.terminal_bound) / \
                ((1. - self.beta_2) * t)

            clipped_lr_t = K.minimum(
                K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
            p_t = p - clipped_lr_t * m_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 66
Project: Coloring-greyscale-images   Author: emilwallner   File: AdamAccumulate.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr

        completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())

        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * completed_updates))

        t = completed_updates + 1

        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))

        # self.iterations incremented after processing a batch
        # batch:              1 2 3 4 5 6 7 8 9
        # self.iterations:    0 1 2 3 4 5 6 7 8
        # update_switch = 1:        x       x    (if accum_iters=4)  
        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)
        update_switch = K.cast(update_switch, K.floatx())

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]

        if self.amsgrad:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        else:
            vhats = [K.zeros(1) for _ in params]

        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):

            sum_grad = tg + g
            avg_grad = sum_grad / self.accum_iters_float

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)

            if self.amsgrad:
                vhat_t = K.maximum(vhat, v_t)
                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))
            else:
                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))
            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))
            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))
        return self.updates 
Example 67
Project: NTM-Keras   Author: SigmaQuan   File: ntm.py    MIT License 4 votes vote down vote up
def reset_states(self):
        print("begin reset_states(self)")
        assert self.stateful, 'Layer must be stateful.'
        input_shape = self.input_spec[0].shape
        self.depth = 0
        if not input_shape[0]:
            raise Exception('If a RNN is stateful, a complete ' +
                            'input_shape must be provided (including batch size).')
        if hasattr(self, 'states'):
            # K.set_value(self.states[0],
            #             np.zeros((input_shape[0], self.output_dim)))
            # K.set_value(self.states[1],
            #             np.zeros((input_shape[0], self.output_dim)))
            # add by Robot Steven ****************************************#
            # previous inner memory
            K.set_value(self.states[0],
                        np.zeros((input_shape[0], self.controller_output_dim)))
            # previous inner cell
            K.set_value(self.states[1],
                        np.zeros((input_shape[0], self.controller_output_dim)))
            # previous memory
            K.set_value(self.states[2],
                        np.zeros((input_shape[0], self.memory_dim * self.memory_size)))
            # K.set_value(self.states[2],
            #             np.zeros((input_shape[0], self.memory_size, self.memory_dim)))
            # previous writing addresses
            K.set_value(self.states[3],
                        np.zeros((input_shape[0], self.num_write_head * self.memory_size)))
            # K.set_value(self.states[3],
            #             np.zeros((input_shape[0], self.num_write_head * self.memory_size)))
            # previous reading addresses
            K.set_value(self.states[4],
                        np.zeros((input_shape[0], self.num_read_head * self.memory_size)))
            # previous reading content
            K.set_value(self.states[5],
                        np.zeros((input_shape[0], self.num_read_head * self.memory_dim)))
            # add by Robot Steven ****************************************#
        else:
            # self.states = [K.zeros((input_shape[0], self.output_dim)),
            #                K.zeros((input_shape[0], self.output_dim))]
            # add by Robot Steven ****************************************#
            self.states = [K.zeros((input_shape[0], self.controller_output_dim)),  # h_tm1
                           K.zeros((input_shape[0], self.controller_output_dim)),  # c_tm1]
                           K.zeros((input_shape[0], self.memory_dim * self.memory_size)),
                           # K.zeros((input_shape[0], self.memory_size, self.memory_dim)),
                           K.zeros((input_shape[0], self.num_write_head * self.memory_size)),
                           K.zeros((input_shape[0], self.num_read_head * self.memory_size)),
                           K.zeros((input_shape[0], self.num_read_head * self.memory_dim))]
            # add by Robot Steven ****************************************#
        print("end reset_states(self)\n") 
Example 68
Project: keras-optimizers   Author: hi-im-ryanli   File: frankenstein.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        t = K.cast(self.iterations, K.floatx()) + 1

        # Due to the recommendations in [2], i.e. warming momentum schedule
        momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
        momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
        m_schedule_new = self.m_schedule * momentum_cache_t
        m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
        self.updates.append((self.m_schedule, m_schedule_new))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
            # the following equations given in [1]
            g_prime = g / (1. - m_schedule_new)
            m_t = self.beta_1 * m + (1. - self.beta_1) * g
            m_t_prime = m_t / (1. - m_schedule_next)
            if np.random.choice([1, -1]) == 1:
                v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
            else:
                v_t = K.maximum(self.beta_2 * v, K.abs(g))
            v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
            m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))

            p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 69
Project: keras-optimizers   Author: hi-im-ryanli   File: frankenstein.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        shapes = [K.int_shape(p) for p in params]
        accumulators = [K.zeros(shape) for shape in shapes]
        delta_accumulators = [K.zeros(shape) for shape in shapes]

        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr *= (1. / (1. + self.decay * K.cast(self.iterations,
                                                  K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]

        self.weights = [self.iterations] + ms + vs

        for p, g, m, v, a, d_a in zip(params, grads, ms, vs, accumulators, delta_accumulators):
            # update accumulator
            new_a = self.rho * a + (1. - self.rho) * K.square(g)
            self.updates.append(K.update(a, new_a))

            # use the new accumulator and the *old* delta_accumulator
            update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * update
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(update)
            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 70
Project: CapsNet-Collections   Author: Jiankai-Sun   File: capsulelayers.py    GNU General Public License v3.0 4 votes vote down vote up
def call(self, inputs, training=None):
        # inputs.shape=[None, input_num_capsule, input_dim_vector]
        # Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector]
        inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2)

        # Replicate num_capsule dimension to prepare being multiplied by W
        # Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector]
        inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1])

        """  
        # Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size.
        # Now W has shape  = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector]
        w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1])
        
        # Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
        inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3])
        """
        # Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow.
        # inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
        inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]),
                             elems=inputs_tiled,
                             initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector]))
        """
        # Routing algorithm V1. Use tf.while_loop in a dynamic way.
        def body(i, b, outputs):
            c = tf.nn.softmax(self.bias, dim=2)  # dim=2 is the num_capsule dimension
            outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
            b = b + K.sum(inputs_hat * outputs, -1, keepdims=True)
            return [i-1, b, outputs]

        cond = lambda i, b, inputs_hat: i > 0
        loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)]
        _, _, outputs = tf.while_loop(cond, body, loop_vars)
        """
        # Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance
        assert self.num_routing > 0, 'The num_routing should be > 0.'
        for i in range(self.num_routing):
            c = tf.nn.softmax(self.bias, dim=2)  # dim=2 is the num_capsule dimension
            # outputs.shape=[None, 1, num_capsule, 1, dim_vector]
            outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))

            # last iteration needs not compute bias which will not be passed to the graph any more anyway.
            if i != self.num_routing - 1:
                # self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True))
                self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True)
            # tf.summary.histogram('BigBee', self.bias)  # for debugging
        return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector]) 
Example 71
Project: conv_qsar_fast   Author: connorcoley   File: GraphEmbedding_sumAfter.py    MIT License 4 votes vote down vote up
def get_output_singlesample(self, original_graph):
		'''For a 3D tensor, get the output. Avoids the need for even more complicated vectorization'''
		# Check padding
		if self.padding:
			rowsum = original_graph.sum(axis = 0) # add across
			trim = rowsum[:, -1] # last feature == bond flag
			trim_to = T.eq(trim, 0).nonzero()[0][0] # first index with no bonds
			original_graph = original_graph[:trim_to, :trim_to, :] # reduced graph

		# Get attribute values for layer zero
		# where attributes is a 2D tensor and attributes[#, :] is the vector of
		# concatenated node and edge attributes. In the first layer (depth 0), the 
		# edge attribute section is initialized to zeros. After increasing depth, howevevr,
		# this part of the vector will become non-zero.

		# The first attributes matrix is just graph_tensor[i, i, :], but we can't use that 
		# kind of advanced indexing
		# Want to extract tensor diagonal as matrix, but can't do that directly...
		# Want to loop over third dimension, so need to dimshuffle
		(attributes, updates) = theano.scan(lambda x: x.diagonal(), sequences = original_graph.dimshuffle((2, 0, 1)))
		attributes.name = 'attributes'
		# Now the attributes is (N_features x N_nodes), so we need to transpose
		attributes = attributes.T
		attributes.name = 'attributes post-transpose'

		# Get initial fingerprint
		presum_fp = self.attributes_to_fp_contribution(attributes, 0)
		fp = K.sum(presum_fp, axis = 0) # sum across atom contributions
		fp.name = 'initial fingerprint'

		# Get bond matrix
		bonds = original_graph[:, :, -1] # flag if the bond is present, (N_atom x N_atom)
		bonds.name = 'bonds'

		graph = original_graph
		# Iterate through different depths, updating attributes each time
		for depth in range(self.depth):
			(attributes, graph) = self.attributes_update(attributes, depth + 1, graph, original_graph, bonds)
			presum_fp_new = self.attributes_to_fp_contribution(attributes, depth + 1)
			presum_fp_new.name = 'presum_fp_new contribution'
			fp = fp + K.sum(presum_fp_new, axis = 0) 

		return fp 
Example 72
Project: conv_qsar_fast   Author: connorcoley   File: GraphEmbedding.py    MIT License 4 votes vote down vote up
def get_output_singlesample(self, original_graph):
		'''For a 3D tensor, get the output. Avoids the need for even more complicated vectorization'''
		# Check padding
		if self.padding:
			rowsum = original_graph.sum(axis = 0) # add across
			trim = rowsum[:, -1] # last feature == bond flag
			trim_to = T.eq(trim, 0).nonzero()[0][0] # first index with no bonds
			original_graph = original_graph[:trim_to, :trim_to, :] # reduced graph

		# Get attribute values for layer zero
		# where attributes is a 2D tensor and attributes[#, :] is the vector of
		# concatenated node and edge attributes. In the first layer (depth 0), the 
		# edge attribute section is initialized to zeros. After increasing depth, howevevr,
		# this part of the vector will become non-zero.

		# The first attributes matrix is just graph_tensor[i, i, :], but we can't use that 
		# kind of advanced indexing
		# Want to extract tensor diagonal as matrix, but can't do that directly...
		# Want to loop over third dimension, so need to dimshuffle
		(attributes, updates) = theano.scan(lambda x: x.diagonal(), sequences = original_graph.dimshuffle((2, 0, 1)))
		attributes.name = 'attributes'
		# Now the attributes is (N_features x N_nodes), so we need to transpose
		attributes = attributes.T
		attributes.name = 'attributes post-transpose'

		# Get initial fingerprint
		presum_fp = self.attributes_to_fp_contribution(attributes, 0)
		fp = K.sum(presum_fp, axis = 0) # sum across atom contributions
		fp.name = 'initial fingerprint'

		# Get bond matrix
		bonds = original_graph[:, :, -1] # flag if the bond is present, (N_atom x N_atom)
		bonds.name = 'bonds'

		# Iterate through different depths, updating attributes each time
		graph = original_graph
		for depth in range(self.depth):
			(attributes, graph) = self.attributes_update(attributes, depth + 1, graph, original_graph, bonds)
			presum_fp_new = self.attributes_to_fp_contribution(attributes, depth + 1)
			presum_fp_new.name = 'presum_fp_new contribution'
			fp = fp + K.sum(presum_fp_new, axis = 0) 

		return fp 
Example 73
Project: pyDNAbinding   Author: nboley   File: keraslib.py    GNU General Public License v2.0 4 votes vote down vote up
def iter_weighted_batch_samples(model, batch_iterator, oversampling_ratio=1):
    """Iter batches where poorly predicted samples are more frequently selected.

    model: the model to predict from
    batch_iterator: batch iterator to draw samples from
    oversampling_ratio: how many batch_iterator batches to sample from
    """
    assert oversampling_ratio > 0

    while True:
        # group the output of oversampling_ratio batches together
        all_batches = defaultdict(list)
        for i in xrange(oversampling_ratio):
            for key, val in next(batch_iterator).iteritems():
                all_batches[key].append(val)
        # find the batch size
        batch_size = next(all_batches.itervalues())[0].shape[0]
        # stack the output
        for key, batches in all_batches.iteritems():
            all_batches[key] = np.vstack(batches)

        # weight the batch values
        pred_vals = model.predict_on_batch(all_batches)
        weights = np.zeros(batch_size*oversampling_ratio)
        for key in pred_vals:
            if key.endswith('binding_occupancies'): continue
            losses = (pred_vals[key] - all_batches[key])**2
            # set ambiguous labels to 0
            losses[all_batches[key] < -0.5] = 0
            # add to the losses,
            inner_weights = losses.sum(1)
            # re-weight the rows with ambiguous labels
            inner_weights *= losses.shape[1]/((
                all_batches[key] > -0.5).sum(1) + 1e-6)
            # make sure that every row has some weight
            inner_weights += 1e-6
            weights += inner_weights
        # normalize the weights to 1
        weights /= weights.sum()

        # downsample batch_size observations
        output = {}
        loss_indices = np.random.choice(
            len(weights), size=batch_size, replace=False, p=weights)
        for key in all_batches.keys():
            output[key] = all_batches[key][loss_indices,:]
        yield output
    
    return 
Example 74
Project: FormicID   Author: naturalis   File: optimizer.py    MIT License 4 votes vote down vote up
def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)

        self.updates = [K.update_add(self.iterations, 1)]
        t = self.iterations + 1

        shapes = [K.get_variable_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]

        loss_prev = K.variable(0)
        self.updates.append(K.update(loss_prev, loss))

        # Calculate the numerator of the Eve coefficient
        d_num_t = K.abs(loss_prev - loss)
        self.updates.append(K.update(self.d_num, d_num_t))

        # Calculate the denominator of the Eve coefficient
        d_den_t = K.abs(K.minimum(loss_prev, loss) - self.loss_min)
        self.updates.append(K.update(self.d_den, d_den_t))

        # Calculate the Eve coefficient. At the first iteration, it is 1.
        d_tilde_t = K.clip(
            (d_num_t + self.fmin_pos) / (d_den_t + self.fmin_pos),
            1. / self.c,
            self.c,
        )
        d_t = (self.beta_3 * self.d) + (1. - self.beta_3) * d_tilde_t
        d_t = K.switch(K.greater(t, 1), d_t, K.constant(1))
        self.updates.append(K.update(self.d, d_t))

        # Calculate the effective learning rate as lr / (d * decay)
        lr_eff_t = self.lr / (d_t * (1. + (self.iterations * self.decay)))
        self.updates.append(K.update(self.lr_eff, lr_eff_t))

        # Apply bias correction to the learning rate
        lr_hat_t = (
            lr_eff_t
            * K.sqrt(1. - K.pow(self.beta_2, t))
            / (1. - K.pow(self.beta_1, t))
        )

        # Update per parameter
        for p, g, m, v in zip(params, grads, ms, vs):
            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            self.updates.append(K.update(m, m_t))

            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
            self.updates.append(K.update(v, v_t))

            p_t = p - lr_hat_t * m_t / (K.sqrt(v_t) + self.epsilon)
            new_p = p_t
            # Apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)
            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 75
Project: MCLNN   Author: fadymedhat   File: layers.py    MIT License 4 votes vote down vote up
def build(self, input_shape):
        assert len(input_shape) == 3
        input_dim = input_shape[2]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_shape[1], input_dim))]

        self.W = self.init((self.order * 2 + 1, input_dim, self.output_dim),
                           name='{}_W'.format(self.name))
        # self.W = self.print_dim('W :',self.W)
        self.b = K.zeros((self.output_dim,),
                         name='{}_b'.format(self.name))

        self.trainable_weights = [self.W, self.b]

        self.weightmask = self.construct_mask(feature_count=input_dim,
                                              hidden_count=self.output_dim,
                                              bandwidth=self.bandwidth,
                                              overlap=self.overlap,
                                              layer_is_masked=self.layer_is_masked)

        # --K_START -- Refer to keras documentation for the below.
        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)

        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.activity_regularizer:
            self.activity_regularizer.set_layer(self)
            self.regularizers.append(self.activity_regularizer)

        self.constraints = {}
        if self.W_constraint:
            self.constraints[self.W] = self.W_constraint
        if self.b_constraint:
            self.constraints[self.b] = self.b_constraint

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        # --K_START -- Refer to keras documentation for the above. 
Example 76
Project: keras-adabound   Author: titu1994   File: adabound.py    MIT License 4 votes vote down vote up
def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
                                                      K.dtype(self.decay))))

        t = K.cast(self.iterations, K.floatx()) + 1

        # Applies bounds on actual learning rate
        step_size = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                          (1. - K.pow(self.beta_1, t)))

        final_lr = self.final_lr * lr / self.base_lr
        lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.))
        upper_bound = final_lr * (1. + 1. / (self.gamma * t))

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        if self.amsbound:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        else:
            vhats = [K.zeros(1) for _ in params]
        self.weights = [self.iterations] + ms + vs + vhats

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            # apply weight decay
            if self.weight_decay != 0.:
                g += self.weight_decay * K.stop_gradient(p)

            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)

            if self.amsbound:
                vhat_t = K.maximum(vhat, v_t)
                denom = (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, vhat_t))
            else:
                denom = (K.sqrt(v_t) + self.epsilon)

            # Compute the bounds
            step_size_p = step_size * K.ones_like(denom)
            step_size_p_bound = step_size_p / denom
            bounded_lr_t = m_t * K.minimum(K.maximum(step_size_p_bound,
                                                     lower_bound), upper_bound)

            p_t = p - bounded_lr_t

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            # Apply constraints.
            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 77
Project: CDSGD   Author: SCSLabISU   File: EASGD.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def get_updates(self, params, constraints, loss):
        parameter_mean=globals()["parameter_mean"]
        epoch_count=globals()["epoch_count"]

        copy_shapes = [K.get_variable_shape(p) for p in params]
        parameter_copy = [K.zeros(copy_shape) for copy_shape in copy_shapes]
        parameter_copy=params

        grads = self.get_gradients(loss, parameter_copy)
        self.updates = []

        lr = self.lr
        alpha=self.alpha
        communication_period=self.communication_period
        if self.initial_decay > 0:
            lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
            self.updates .append(K.update_add(self.iterations, 1))

        # momentum
        shapes = [K.get_variable_shape(p) for p in params]
        moments = [K.zeros(shape) for shape in shapes]



        self.weights = [self.iterations] + moments
        temp_param_mean=[]
        for p, pi, pm, g, m in zip(params,parameter_copy, parameter_mean, grads, moments):
        # for i in range(len(params)):
        #     p=params[i]
        #     p1=parameter_copy[i]
        #     pm=parameter_mean[i]
        #     g=grads[i]
        #     m=moments[i]

            # Momentum term
            #v = self.momentum * m - lr * g  # velocity
            v = - lr * g # no momentum

            self.updates.append(K.update(m, v))
            if self.nesterov:
                if (epoch_count%communication_period)==0:
                    new_pm = pm + alpha*(pi-pm)
                    new_p = p - alpha*(pi-pm) + self.momentum * v - lr * g
                    temp_param_mean.append(new_p)
                else:
                    new_p = p + self.momentum * v - lr * g
            else:
                if (epoch_count%communication_period)==0:
                    new_pm = pm + alpha*(pi-pm)
                    new_p = p - alpha*(pi-pm) + v
                    temp_param_mean.append(new_pm)
                else:
                    new_p = p + v
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)

            self.updates.append(K.update(p, new_p))
        parameter_mean=temp_param_mean
        return self.updates 
Example 78
Project: CDSGD   Author: SCSLabISU   File: CDSGD.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def get_updates(self, params, constraints, loss):
        pi=self.pi
        n_agents=self.n_agents
        agent_id=self.agent_id
        parameters=globals()["parameters"]
        info_shapes = [K.get_variable_shape(p) for p in params]
        parameter_copy = [0 for nb in range(n_agents)]
        for nb in range(n_agents):
            parameter_copy[nb]=parameters[nb]
            if (nb==agent_id):
                parameter_copy[nb]=params


        grads = self.get_gradients(loss, params)
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
            lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
            self.updates .append(K.update_add(self.iterations, 1))

        # momentum
        shapes = [K.get_variable_shape(p) for p in params]
        moments = [K.zeros(shape) for shape in shapes]
        dist_accumulator = [K.zeros(shape) for shape in shapes]



        self.weights = [self.iterations] + moments
        #for p, g, m, d, p_agents in zip(params, grads, moments, dist_accumulator, *parameter_copy):
        for i in range(len(params)):
            p=params[i]
            g=grads[i]
            m=moments[i]
            d=dist_accumulator[i]
            # Momentum term
            v = self.momentum * m - lr * g  # velocity
            #v = - lr * g # no momentum
            self.updates.append(K.update(m, v))
            if self.nesterov:
                for nb in range(n_agents):
                    d+=pi[nb][agent_id]*parameter_copy[nb][i]
                new_p = d + self.momentum * v - lr * g
            else:
                # This is for Debug only
                # if count>5:
                #     raise ValueError('parameters: ' + str(p1) + str(p2) + str(p3) + str(p4) + str(p5)  )

                for nb in range(n_agents):
                    d+=pi[nb][agent_id]*parameter_copy[nb][i]
                    #raise ValueError('pi:' + str(K.eval(pi)))
                new_p = d + v
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 79
Project: CDSGD   Author: SCSLabISU   File: CDMSGD.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def get_updates(self, params, constraints, loss):
        pi=self.pi
        n_agents=self.n_agents
        agent_id=self.agent_id
        parameters=globals()["parameters"]
        info_shapes = [K.get_variable_shape(p) for p in params]
        parameter_copy = [0 for i in range(n_agents)]
        for nb in range(n_agents):
            parameter_copy[nb]=parameters[nb]
            if (nb==agent_id):
                parameter_copy[nb]=params


        grads = self.get_gradients(loss, params)
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
            lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
            self.updates .append(K.update_add(self.iterations, 1))

        # momentum
        shapes = [K.get_variable_shape(p) for p in params]
        moments = [K.zeros(shape) for shape in shapes]
        dist_accumulator = [K.zeros(shape) for shape in shapes]



        self.weights = [self.iterations] + moments
        #for p, g, m, d, p_agents in zip(params, grads, moments, dist_accumulator, *parameter_copy):
        for i in range(len(params)):
            p=params[i]
            g=grads[i]
            m=moments[i]
            d=dist_accumulator[i]
            # Momentum term
            #v = self.momentum * m - lr * g  # velocity
            v = self.momentum * m - lr * g  # velocity
            self.updates.append(K.update(m, v))
            if self.nesterov:
                for nb in range(n_agents):
                    d+=pi[nb][agent_id]*parameter_copy[nb][i]
                new_p = d + self.momentum * v - lr * g
            else:
                # This is for Debug only
                # if count>5:
                #     raise ValueError('parameters: ' + str(p1) + str(p2) + str(p3) + str(p4) + str(p5)  )

                for nb in range(n_agents):
                    d+=pi[nb][agent_id]*parameter_copy[nb][i]
                new_p = d + v
            # apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates 
Example 80
Project: dynamic_memory_networks_with_keras   Author: vchudinov   File: attention_cells.py    GNU General Public License v3.0 4 votes vote down vote up
def __init__(self,
                 units,
                 activation='sigmoid',
                 batch_size=0,
                 recurrent_activation='hard_sigmoid',
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 recurrent_initializer='orthogonal',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 recurrent_regularizer=None,
                 bias_regularizer=None,
                 kernel_constraint=None,
                 recurrent_constraint=None,
                 bias_constraint=None,
                 dropout=0.,
                 recurrent_dropout=0.,
                 implementation=1,
                 return_sequences=False,
                 **kwargs):
        """Identical to keras.recurrent.GRUCell.
        The difference comes from the computation in self.call
        """


        super(SoftAttnGRU, self).__init__(**kwargs)
        self.batch_size = batch_size
        self.units = units
        self.return_sequences = return_sequences
        self.activation = activations.get(activation)
        self.recurrent_activation = activations.get(recurrent_activation)
        self.use_bias = use_bias

        self.kernel_initializer = initializers.get(kernel_initializer)
        self.recurrent_initializer = initializers.get(recurrent_initializer)
        self.bias_initializer = initializers.get(bias_initializer)

        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)

        self.kernel_constraint = constraints.get(kernel_constraint)
        self.recurrent_constraint = constraints.get(recurrent_constraint)
        self.bias_constraint = constraints.get(bias_constraint)

        self.dropout = min(1., max(0., dropout))
        self.recurrent_dropout = min(1., max(0., recurrent_dropout))
        self.implementation = implementation
        self.state_size = self.units
        self._dropout_mask = None
        self._recurrent_dropout_mask = None

        self._input_map = {}

        super(SoftAttnGRU, self).__init__(**kwargs)