Python keras.backend.zeros() Examples
The following are code examples for showing how to use keras.backend.zeros(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.
Example 1
| Project: CapsNet Author: l11x0m7 File: capsule.py MIT License | 6 votes |
|
def call(self, inputs, **kwargs):
# (batch_size, 1, input_num_capsule, input_dim_capsule)
expand_inputs = K.expand_dims(inputs, axis=1)
# (batch_size, num_capsule, input_num_capsule, input_dim_capsule)
expand_inputs = K.tile(expand_inputs, (1, self.num_capsule, 1, 1))
# (batch_size, num_capsule, input_num_capsule, dim_capsule)
u_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, axes=[2, 3]), expand_inputs)
if self.num_routing <= 0:
self.num_routing = 3
# (batch_size, num_capsule, input_num_capsule)
b = K.zeros((K.shape(u_hat)[0], self.num_capsule, self.input_num_capsule))
for i in xrange(self.num_routing):
# (batch_size, num_capsule, input_num_capsule)
c = softmax(b, axis=1)
# (batch_size, num_capsule, dim_capsule)
s = K.batch_dot(c, u_hat, axes=[2, 2])
squashed_s = squash(s)
if i < self.num_routing - 1:
# (batch_size, num_capsule, input_num_capsule)
b += K.batch_dot(squashed_s, u_hat, axes=[2, 3])
return squashed_s
Example 2
| Project: dockerizeme Author: dockerizeme File: snippet.py Apache License 2.0 | 6 votes |
|
def build(self, input_shape):
assert len(input_shape) >= 3
self.input_spec = [InputSpec(shape=input_shape)]
nb_samples, nb_time, input_dim = input_shape
if not self.layer.built:
self.layer.build(input_shape)
self.layer.built = True
super(Attention, self).build()
self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name))
self.W2 = self.layer.init((self.layer.output_dim, input_dim), name='{}_W2'.format(self.name))
self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name))
self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name))
self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name))
self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name))
self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3]
Example 3
| Project: Keras-DropBlock Author: MLearing File: drop_block.py MIT License | 6 votes |
|
def _compute_valid_seed_region(self, seq_length):
positions = K.arange(seq_length)
half_block_size = self.block_size // 2
valid_seed_region = K.switch(
K.all(
K.stack(
[
positions >= half_block_size,
positions < seq_length - half_block_size,
],
axis=-1,
),
axis=-1,
),
K.ones((seq_length,)),
K.zeros((seq_length,)),
)
return K.expand_dims(K.expand_dims(valid_seed_region, axis=0), axis=-1)
Example 4
| Project: Keras-DropBlock Author: MLearing File: drop_block.py MIT License | 6 votes |
|
def _compute_valid_seed_region(self, height, width):
positions = K.concatenate([
K.expand_dims(K.tile(K.expand_dims(K.arange(height), axis=1), [1, width]), axis=-1),
K.expand_dims(K.tile(K.expand_dims(K.arange(width), axis=0), [height, 1]), axis=-1),
], axis=-1)
half_block_size = self.block_size // 2
valid_seed_region = K.switch(
K.all(
K.stack(
[
positions[:, :, 0] >= half_block_size,
positions[:, :, 1] >= half_block_size,
positions[:, :, 0] < height - half_block_size,
positions[:, :, 1] < width - half_block_size,
],
axis=-1,
),
axis=-1,
),
K.ones((height, width)),
K.zeros((height, width)),
)
return K.expand_dims(K.expand_dims(valid_seed_region, axis=0), axis=-1)
Example 5
| Project: group-ksparse-temporal-cnns Author: srph25 File: ops.py MIT License | 6 votes |
|
def diff(a, n=1, axis=-1):
if axis == -1:
axis = K.ndim(a) - 1
a_aug = K.tile(K.expand_dims(a, axis=1), [1, n] + [1] * (K.ndim(a) - 1))
pattern = (axis,) + tuple(set(range(K.ndim(a))) - {axis})
inv_pattern = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, K.ndim(a)))
def step(inputs, states):
prev_output = states[0]
t = states[1]
t_int = K.cast(t[0], 'int32')
prev_output_aug = K.permute_dimensions(prev_output, pattern)
inputs_aug = K.permute_dimensions(inputs, pattern)
output_aug = K.switch(K.all(t_int > 0),
K.concatenate([inputs_aug[:t_int], prev_output_aug[1:] - prev_output_aug[:-1]], axis=0),
K.concatenate([inputs_aug[:1], inputs_aug[1:] - inputs_aug[:-1]], axis=0))
output = K.permute_dimensions(output_aug, inv_pattern)
return output, [output, t + 1]
d_aug = K.permute_dimensions(K.rnn(step, a_aug, [K.zeros_like(a), K.zeros((1,))])[0], pattern)
d = K.permute_dimensions(d_aug[-(K.shape(a)[axis] - n):], inv_pattern)
return d
Example 6
| Project: group-ksparse-temporal-cnns Author: srph25 File: ops.py MIT License | 6 votes |
|
def cumsum(a, n=1, axis=-1):
if axis == -1:
axis = K.ndim(a) - 1
a_aug = K.tile(K.expand_dims(a, axis=1), [1, n] + [1] * (K.ndim(a) - 1))
pattern = (axis,) + tuple(set(range(K.ndim(a))) - {axis})
inv_pattern = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, K.ndim(a)))
def step(inputs, states):
prev_output = states[0]
t = states[1]
t_int = K.cast(t[0], 'int32')
prev_output_aug = K.permute_dimensions(prev_output, pattern)
inputs_aug = K.permute_dimensions(inputs, pattern)
output_aug = K.switch(K.all(t_int > 0), K.cumsum(prev_output_aug, axis=0), K.cumsum(inputs_aug, axis=0))
output = K.permute_dimensions(output_aug, inv_pattern)
return output, [output, t + 1]
c_aug = K.permute_dimensions(K.rnn(step, a_aug, [K.zeros_like(a), K.zeros((1,))])[0], pattern)
c = K.permute_dimensions(c_aug[-(K.shape(a)[axis] - n):], inv_pattern)
return c
Example 7
| Project: keras_bn_library Author: bnsnapper File: rnnrbm.py MIT License | 6 votes |
|
def reset_states(self):
assert self.stateful, 'Layer must be stateful.'
input_shape = self.input_spec[0].shape
if not input_shape[0]:
raise Exception('If a RNN is stateful, a complete ' +
'input_shape must be provided (including batch size).')
if hasattr(self, 'states'):
K.set_value(self.states[0],
np.zeros((input_shape[0], self.hidden_recurrent_dim)))
K.set_value(self.states[1],
np.zeros((input_shape[0], self.input_dim)))
K.set_value(self.states[2],
np.zeros((input_shape[0], self.hidden_dim)))
else:
self.states = [K.zeros((input_shape[0], self.hidden_recurrent_dim)),
K.zeros((input_shape[0], self.input_dim)),
K.zeros((input_shape[0], self.hidden_dim))]
Example 8
| Project: keras_bn_library Author: bnsnapper File: recurrent.py MIT License | 6 votes |
|
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.input_dim = input_shape[2]
self.W = self.init((self.output_dim, 4 * self.input_dim),
name='{}_W'.format(self.name))
self.U = self.inner_init((self.input_dim, 4 * self.input_dim),
name='{}_U'.format(self.name))
self.b = K.variable(np.hstack((np.zeros(self.input_dim),
K.get_value(self.forget_bias_init((self.input_dim,))),
np.zeros(self.input_dim),
np.zeros(self.input_dim))),
name='{}_b'.format(self.name))
self.A = self.init((self.input_dim, self.output_dim),
name='{}_A'.format(self.name))
self.ba = K.zeros((self.output_dim,), name='{}_ba'.format(self.name))
self.trainable_weights = [self.W, self.U, self.b, self.A, self.ba]
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
Example 9
| Project: keras_bn_library Author: bnsnapper File: recurrent.py MIT License | 6 votes |
|
def reset_states(self):
assert self.stateful, 'Layer must be stateful.'
input_shape = self.input_spec[0].shape
if not input_shape[0]:
raise ValueError('If a RNN is stateful, it needs to know '
'its batch size. Specify the batch size '
'of your input tensors: \n'
'- If using a Sequential model, '
'specify the batch size by passing '
'a `batch_input_shape` '
'argument to your first layer.\n'
'- If using the functional API, specify '
'the time dimension by passing a '
'`batch_shape` argument to your Input layer.')
if hasattr(self, 'states'):
K.set_value(self.states[0],
np.zeros((input_shape[0], self.input_dim)))
K.set_value(self.states[1],
np.zeros((input_shape[0], self.output_dim)))
else:
self.states = [K.zeros((input_shape[0], self.input_dim)),
K.zeros((input_shape[0], self.output_dim))]
Example 10
| Project: yoctol-keras-layer-zoo Author: Yoctol File: rnn_cell.py GNU General Public License v3.0 | 6 votes |
|
def reset_states(self, states=None):
if states is None:
self.recurrent_layer.reset_states(states)
else:
self.recurrent_layer.reset_states(states[:-1])
batch_size = self.recurrent_layer.input_spec[0].shape[0]
if self.dense_state is None:
self.dense_state = K.zeros((
batch_size,
self.dense_layer.units
))
elif states is None:
K.set_value(
self.dense_state,
np.zeros((batch_size, self.dense_layer.units))
)
else:
K.set_value(
self.dense_state,
states[-1]
)
Example 11
| Project: ChimeraNet Author: leichtrhino File: models.py MIT License | 6 votes |
|
def loss_deepclustering(T, F, C, D):
def _loss_deepclustering(y_true, y_pred):
Y = K.reshape(y_true, (-1, T*F, C))
V = K.reshape(y_pred, (-1, T*F, D))
Dm = K.expand_dims(K.batch_dot(
Y,
K.sum(Y, axis=(1,)),
axes=(2, 1)
))
Dm = K.switch(Dm > 0, Dm**-0.25, K.zeros(K.shape(Dm)))
DV, DY = Dm * V, Dm * Y
a = K.sum(K.batch_dot(DV, DV, axes=(1, 1))**2, axis=(1, 2))
b = K.sum(K.batch_dot(DV, DY, axes=(1, 1))**2, axis=(1, 2))
c = K.sum(K.batch_dot(DY, DY, axes=(1, 1))**2, axis=(1, 2))
return (a - 2*b + c)
return _loss_deepclustering
Example 12
| Project: keras-lookahead Author: CyberZHG File: optimizers.py MIT License | 6 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = self.learning_rate * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
p_t = lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
self.updates.append(K.update_sub(p, p_t))
return self.updates
Example 13
| Project: RPGOne Author: RTHMaK File: tree_composition_lstm.py Apache License 2.0 | 6 votes |
|
def get_initial_states(self, inputs):
# The initial buffer is sent into the TreeLSTM as a part of the input.
# i.e., inputs is a concatenation of the transitions and the initial buffer.
# (batch_size, buffer_limit, output_dim+1)
# We will now separate the buffer and the transitions and initialize the
# buffer state of the TreeLSTM with the initial buffer value.
# The rest of the input is the transitions, which we do not need now.
# Take the buffer out.
init_h_for_buffer = inputs[:, :, 1:] # (batch_size, buffer_limit, output_dim)
# Initializing all c as zeros.
init_c_for_buffer = K.zeros_like(init_h_for_buffer)
# Each element in the buffer is a concatenation of h and c for the corresponding
# node
init_buffer = K.concatenate([init_h_for_buffer, init_c_for_buffer], axis=-1)
# We need a symbolic all zero tensor of size (samples, stack_limit, 2*output_dim) for
# init_stack The problem is the first dim (samples) is a place holder and not an actual
# value. So we'll use the following trick
temp_state = K.zeros_like(inputs) # (samples, buffer_ops_limit, input_dim)
temp_state = K.tile(K.sum(temp_state, axis=(1, 2)),
(self.stack_limit, 2*self.output_dim, 1)) # (stack_limit, 2*output_dim, samples)
init_stack = K.permute_dimensions(temp_state, (2, 0, 1)) # (samples, stack_limit, 2*output_dim)
return [init_buffer, init_stack]
Example 14
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 6 votes |
|
def __init__(
self,
nb_domains,
domain_len,
init='glorot_uniform',
**kwargs):
self.nb_domains = nb_domains
self.domain_len = domain_len
self.input = K.placeholder(ndim=4)
self.init = lambda x: (
(initializations.get(init)(x), None),
(K.zeros((self.nb_domains,)), None)
)
self.kwargs = kwargs
self.W_shape = None
self.W = None
self.b = None
super(ConvolutionCoOccupancy, self).__init__(**kwargs)
Example 15
| Project: onnx-keras Author: leodestiny File: custom_layers.py MIT License | 6 votes |
|
def call(self, x, mask=None):
b, ch, r, c = x.shape
half_n = self.n // 2 # half the local region
input_sqr = K.square(x) # square the input
extra_channels = K.zeros((b, int(ch) + 2 * half_n, r, c))
input_sqr = K.concatenate(
[extra_channels[:, :half_n, :, :], input_sqr, extra_channels[:, half_n + int(ch):, :, :]], axis=1)
scale = self.k # offset for the scale
norm_alpha = self.alpha / self.n # normalized alpha
for i in range(self.n):
scale += norm_alpha * input_sqr[:, i:i + int(ch), :, :]
scale = scale ** self.beta
x = x / scale
return x
Example 16
| Project: MCLNN Author: fadymedhat File: layers.py MIT License | 6 votes |
|
def construct_mask(self, feature_count, hidden_count, bandwidth, overlap, layer_is_masked):
bw = bandwidth
ov = overlap
l = feature_count
e = hidden_count
a = np.arange(1, bw + 1)
g = np.arange(1, int(np.ceil((l * e) / (l + bw - ov))) + 1)
if layer_is_masked is False:
binary_mask = np.ones([l, e])
else:
mask = np.zeros([l, e])
flat_matrix = mask.flatten('F')
for i in range(len(a)):
for j in range(len(g)):
lx = a[i] + (g[j] - 1) * (l + bw - ov)
if lx <= l * e:
flat_matrix[lx - 1] = 1
binary_mask = np.transpose(flat_matrix.reshape(e, l))
return binary_mask.astype(np.float32)
Example 17
| Project: dynamic_memory_networks_with_keras Author: vchudinov File: attention_cells.py GNU General Public License v3.0 | 6 votes |
|
def get_initial_state(self, inputs):
return K.zeros(shape= [self.batch_size, self.units])
# build an all-zero tensor of shape (samples, output_dim)
# What the hell am I doing here?
# print("--------------------")
# initial_state = K.zeros_like(inputs) # (samples, timesteps, input_dim)
# print(initial_state.shape)
# initial_state = initial_state[:, :, :-1]
#initial_state = K.sum(initial_state, axis=(1, 2)) # (samples,)
#print(initial_state.shape)
#initial_state = K.expand_dims(initial_state) # (samples, 1)
#print(initial_state.shape)
#raise SystemExit
# if hasattr(self.state_size, '__len__'):
# return [K.tile(initial_state, [1, dim])
# for dim in self.state_size]
# else:
# return [K.tile(initial_state, [1, self.state_size])]
Example 18
| Project: FireDetectionCNN Author: dogusyuksel File: Custom_layers.py MIT License | 6 votes |
|
def get_output(self, train):
X = self.get_input(train)
b, ch, r, c = K.shape(X)
half_n = self.n // 2
input_sqr = K.square(X)
extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
input_sqr,
extra_channels[:, half_n + ch:, :, :]],
axis=1)
scale = self.k
for i in range(self.n):
scale += self.alpha * input_sqr[:, i:i + ch, :, :]
scale = scale ** self.beta
return X / scale
Example 19
| Project: FireDetectionCNN Author: dogusyuksel File: Custom_layers.py MIT License | 6 votes |
|
def get_output(self, train):
X = self.get_input(train)
b, ch, r, c = K.shape(X)
half_n = self.n // 2
input_sqr = K.square(X)
extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
input_sqr,
extra_channels[:, half_n + ch:, :, :]],
axis=1)
scale = self.k
for i in range(self.n):
scale += self.alpha * input_sqr[:, i:i + ch, :, :]
scale = scale ** self.beta
return X / scale
Example 20
| Project: VisualNN Author: angelhunt File: lrn.py GNU General Public License v3.0 | 5 votes |
|
def get_output(self, train):
X = self.get_input(train)
b, ch, r, c = K.shape(X)
half_n = self.n // 2
input_sqr = K.square(X)
extra_channels = K.zeros((b, ch + 2 * half_n, r, c))
input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],
input_sqr,
extra_channels[:, half_n + ch:, :, :]],
axis=1)
scale = self.k
for i in range(self.n):
scale += self.alpha * input_sqr[:, i:i + ch, :, :]
scale = scale ** self.beta
return X / scale
Example 21
| Project: deep-models Author: LaurentMazare File: rhn.py Apache License 2.0 | 5 votes |
|
def reset_states(self):
assert self.stateful, 'Layer must be stateful.'
input_shape = self.input_spec[0].shape
if not input_shape[0]:
raise Exception('If a RNN is stateful, a complete ' +
'input_shape must be provided (including batch size).')
if hasattr(self, 'states'):
K.set_value(self.states[0],
np.zeros((input_shape[0], self.output_dim)))
else:
self.states = [K.zeros((input_shape[0], self.output_dim))]
Example 22
| Project: deep-models Author: LaurentMazare File: lstm_ln.py Apache License 2.0 | 5 votes |
|
def build(self, input_shape):
super(LSTM_LN, self).build(input_shape)
self.gs, self.bs = [], []
for i in xrange(3):
f = 1 if i == 2 else 4
self.gs += [ K.ones((f*self.output_dim,), name='{}_g%i'.format(self.name, i)) ]
self.bs += [ K.zeros((f*self.output_dim,), name='{}_b%d'.format(self.name, i)) ]
self.trainable_weights += self.gs + self.bs
Example 23
| Project: DeepLearningMugenKnock Author: yoyoyo-yo File: cgan_cifar10_keras.py MIT License | 5 votes |
|
def test():
# load trained model
g, _ = G_model()
g.load_weights('cgan_cifar10_keras.h5', by_name=True)
np.random.seed(100)
labels = ["air¥nplane", "auto¥bmobile", "bird", "cat", "deer",
"dog", "frog", "horse", "ship", "truck"]
con_x = np.zeros([10, num_classes])
con_x[np.arange(10), np.arange(num_classes)] = 1
for i in range(3):
input_noise = np.random.uniform(-1, 1, size=(10, 100))
g_output = g.predict(x={"x":input_noise, "con_x": con_x}, verbose=0)
g_output = (g_output + 1 ) / 2
for i in range(10):
gen = g_output[i]
if channel == 1:
gen = gen[..., 0]
cmap = "gray"
elif channel == 3:
cmap = None
plt.subplot(1,10,i+1)
plt.title(labels[i])
plt.imshow(gen, cmap=cmap)
plt.axis('off')
plt.show()
Example 24
| Project: DeepLearningMugenKnock Author: yoyoyo-yo File: cgan_mnist_keras.py MIT License | 5 votes |
|
def test():
# load trained model
g, _ = G_model()
g.load_weights('cgan_cifar10_keras.h5', by_name=True)
np.random.seed(100)
con_x = np.zeros([10, num_classes])
con_x[np.arange(10), np.arange(num_classes)] = 1
for i in range(3):
input_noise = np.random.uniform(-1, 1, size=(10, 100))
g_output = g.predict(x={"x":input_noise, "con_x": con_x}, verbose=0)
g_output = (g_output + 1 ) / 2
for i in range(10):
gen = g_output[i]
if channel == 1:
gen = gen[..., 0]
cmap = "gray"
elif channel == 3:
cmap = None
plt.subplot(1,10,i+1)
plt.title(str(i))
plt.imshow(gen, cmap=cmap)
plt.axis('off')
plt.show()
Example 25
| Project: fancy-cnn Author: textclf File: convolutions.py MIT License | 5 votes |
|
def build(self):
stack_size = self.input_shape[2]
dtensor5 = T.TensorType('float32', (False,)*5)
self.input = dtensor5()
self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
self.W = self.init(self.W_shape)
self.b = shared_zeros((self.nb_filter,))
self.params = [self.W, self.b]
self.regularizers = []
if self.W_regularizer:
self.W_regularizer.set_param(self.W)
self.regularizers.append(self.W_regularizer)
if self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
if self.activity_regularizer:
self.activity_regularizer.set_layer(self)
self.regularizers.append(self.activity_regularizer)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
Example 26
| Project: keras-attention-augmented-convs Author: titu1994 File: attn_augconv.py MIT License | 5 votes |
|
def rel_to_abs(self, x):
shape = K.shape(x)
shape = [shape[i] for i in range(3)]
B, Nh, L, = shape
col_pad = K.zeros(K.stack([B, Nh, L, 1]))
x = K.concatenate([x, col_pad], axis=3)
flat_x = K.reshape(x, [B, Nh, L * 2 * L])
flat_pad = K.zeros(K.stack([B, Nh, L - 1]))
flat_x_padded = K.concatenate([flat_x, flat_pad], axis=2)
final_x = K.reshape(flat_x_padded, [B, Nh, L + 1, 2 * L - 1])
final_x = final_x[:, :, :L, L - 1:]
return final_x
Example 27
| Project: keras-lamb Author: CyberZHG File: optimizer.py MIT License | 5 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = K.cast(self.iterations, K.floatx()) + 1
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * t))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
mhat_t = m_t / (1. - K.pow(self.beta_1, t))
vhat_t = v_t / (1. - K.pow(self.beta_2, t))
u_t = mhat_t / K.sqrt(vhat_t + self.epsilon) + self.weight_decay * p
trust_ratio = K.sqrt(K.sum(K.square(p)) / K.sum(K.square(u_t)))
trust_ratio = K.minimum(K.maximum(trust_ratio, self.lower_bound), self.upper_bound)
lr_p = trust_ratio * lr
new_p = p - lr_p * u_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 28
| Project: 360_aware_saliency Author: MikhailStartsev File: attentive_convlstm.py GNU General Public License v3.0 | 5 votes |
|
def get_initial_states(self, x):
initial_state = K.sum(x, axis=1)
initial_state = K.conv2d(initial_state, K.zeros((self.nb_filters_out, self.nb_filters_in, 1, 1)), border_mode='same')
initial_states = [initial_state for _ in range(len(self.states))]
return initial_states
Example 29
| Project: CapsNet-Fashion-MNIST Author: subarnop File: capsulelayers.py GNU General Public License v3.0 | 5 votes |
|
def __init__(self, num_capsule, dim_vector, num_routing=3,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
**kwargs):
super(CapsuleLayer, self).__init__(**kwargs)
self.num_capsule = num_capsule
self.dim_vector = dim_vector
self.num_routing = num_routing
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
Example 30
| Project: keras-adamw Author: OverLordGoldDragon File: optimizers_v2.py MIT License | 5 votes |
|
def _resource_apply_dense(self, grad, var):
var_dtype = var.dtype.base_dtype
lr_t = self._decayed_lr(var_dtype)
momentum = array_ops.identity(self._get_hyper('momentum', var_dtype))
if K_eval(momentum) != 0:
m = self.get_slot(var, 'momentum')
else:
m = K.zeros(K.int_shape(var))
total_iterations = self.total_iterations
# Learning rate multipliers
if self.lr_multipliers is not None:
lr_t = _apply_lr_multiplier(self, lr_t, var)
# Cosine annealing
if self.use_cosine_annealing and total_iterations != 0:
self.eta_t = _compute_eta_t(self)
v = momentum * m - self.eta_t*lr_t * grad # velocity
m = state_ops.assign(m, v, use_locking=self._use_locking)
if self.nesterov:
var_t = math_ops.sub(var, -momentum * v + self.eta_t*lr_t * grad)
else:
var_t = var + v
# Weight decays
if var.name in self.weight_decays.keys() and total_iterations != 0:
var_t = _apply_weight_decays(self, var, var_t)
iteration_done = self._updates_processed == (self._updates_per_iter - 1)
_up = self._updates_processed
self._updates_processed = (_up + 1) if not iteration_done else 0
if iteration_done and not self._init_notified:
self._init_notified = True
var_update = state_ops.assign(var, var_t, use_locking=self._use_locking)
t_cur = state_ops.assign_add(self.t_cur, int(iteration_done),
use_locking=self._use_locking)
updates = [var_update, m, t_cur]
return control_flow_ops.group(*updates)
Example 31
| Project: gccaps Author: tqbl File: capsules.py MIT License | 5 votes |
|
def __init__(self, n_capsules, dim_capsule, routings=3, use_bias=False,
kernel_initializer='glorot_uniform', bias_initializer='zeros',
**kwargs):
super(CapsuleLayer, self).__init__(**kwargs)
self.n_capsules = n_capsules
self.dim_capsule = dim_capsule
self.routings = routings
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
Example 32
| Project: gccaps Author: tqbl File: capsules.py MIT License | 5 votes |
|
def call(self, inputs, training=None):
"""Apply transformation followed by capsule routing."""
# Create dimension for output capsules and tile along this dim
# (None, *n_capsules*, n_input_capsules, dim_input_capsules)
inputs_tiled = K.tile(K.expand_dims(inputs, 1),
[1, self.n_capsules, 1, 1])
# Apply linear transformation to compute prediction vectors
inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]),
elems=inputs_tiled)
# Add bias to prediction vectors if specified
if self.use_bias:
inputs_hat = K.bias_add(inputs_hat, self.bias,
data_format='channels_first')
# Initialize logit variables to zero
b = K.zeros(shape=[K.shape(inputs_hat)[0],
self.n_capsules,
self.n_input_capsules])
# Apply routing algorithm
for i in range(self.routings):
# Compute coupling coefficients
c = K.softmax(b, axis=1)
# Apple squashing function
outputs = squash(K.batch_dot(c, inputs_hat, [2, 2]))
# Update logits by computing agreement
if i < self.routings - 1:
b += K.batch_dot(outputs, inputs_hat, [2, 3])
return outputs
Example 33
| Project: dockerizeme Author: dockerizeme File: snippet.py Apache License 2.0 | 5 votes |
|
def build(self, input_shape):
# input_dim = input_shape[2]
# self.W_shape = (self.nb_filter, input_dim, self.filter_length, 1)
# self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
if self.bias:
self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
self.trainable_weights = [self.b]
# else:
# self.trainable_weights = [self.W]
self.regularizers = []
#
# if self.W_regularizer:
# self.W_regularizer.set_param(self.W)
# self.regularizers.append(self.W_regularizer)
#
if self.bias and self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
#
# if self.activity_regularizer:
# self.activity_regularizer.set_layer(self)
# self.regularizers.append(self.activity_regularizer)
#
# self.constraints = {}
# if self.W_constraint:
# self.constraints[self.W] = self.W_constraint
if self.bias and self.b_constraint:
self.constraints[self.b] = self.b_constraint
#
# if self.initial_weights is not None:
# self.set_weights(self.initial_weights)
# del self.initial_weights
Example 34
| Project: dockerizeme Author: dockerizeme File: snippet.py Apache License 2.0 | 5 votes |
|
def build(self, input_shape):
if self.dim_ordering == 'th':
stack_size = input_shape[1]
self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
elif self.dim_ordering == 'tf':
stack_size = input_shape[3]
self.W_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
# self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
if self.bias:
self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
self.trainable_weights = [self.b]
# else:
# self.trainable_weights = [self.W]
self.regularizers = []
# if self.W_regularizer:
# self.W_regularizer.set_param(self.W)
# self.regularizers.append(self.W_regularizer)
if self.bias and self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
if self.activity_regularizer:
self.activity_regularizer.set_layer(self)
self.regularizers.append(self.activity_regularizer)
self.constraints = {}
# if self.W_constraint:
# self.constraints[self.W] = self.W_constraint
if self.bias and self.b_constraint:
self.constraints[self.b] = self.b_constraint
# if self.initial_weights is not None:
# self.set_weights(self.initial_weights)
# del self.initial_weights
Example 35
| Project: keras_bn_library Author: bnsnapper File: recurrent.py MIT License | 5 votes |
|
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
input_dim = input_shape[2]
self.input_dim = input_dim
if self.stateful:
self.reset_states()
else:
self.states = [None, None]
self.states_dim = [self.input_dim, self.output_dim]
self.weight_size = self.output_dim * 4
self.W = self.add_weight((input_dim, self.weight_size),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer)
self.U = self.add_weight((input_dim, self.weight_size),
initializer=self.inner_init,
name='{}_U'.format(self.name),
regularizer=self.U_regularizer)
def b_reg(shape, name=None):
return K.variable(np.hstack((np.zeros(self.output_dim),
K.get_value(self.forget_bias_init((self.output_dim,))),
np.zeros(self.output_dim),
np.zeros(self.output_dim))),
name='{}_b'.format(self.name))
self.b = self.add_weight((self.weight_size,),
initializer=b_reg,
name='{}_b'.format(self.name),
regularizer=self.b_regularizer)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
self.built = True
Example 36
| Project: deep-pmsm Author: wkirgsn File: custom_layers.py MIT License | 5 votes |
|
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):
"""Layer normalization layer
See: [Layer Normalization](https://arxiv.org/pdf/1607.06450.pdf)
:param center: If True, add offset of `beta` to normalized tensor. If False, `beta` is ignored.
:param scale: If True, multiply by `gamma`. If False, `gamma` is not used.
:param epsilon: Epsilon for variance.
:param gamma_initializer: Initializer for the gamma weight.
:param beta_initializer: Initializer for the beta weight.
:param gamma_regularizer: Optional regularizer for the gamma weight.
:param beta_regularizer: Optional regularizer for the beta weight.
:param gamma_constraint: Optional constraint for the gamma weight.
:param beta_constraint: Optional constraint for the beta weight.
:param kwargs:
"""
super(LayerNormalization, self).__init__(**kwargs)
self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = initializers.get(gamma_initializer)
self.beta_initializer = initializers.get(beta_initializer)
self.gamma_regularizer = regularizers.get(gamma_regularizer)
self.beta_regularizer = regularizers.get(beta_regularizer)
self.gamma_constraint = constraints.get(gamma_constraint)
self.beta_constraint = constraints.get(beta_constraint)
self.gamma, self.beta = None, None
Example 37
| Project: NTM-Keras Author: SigmaQuan File: lstm2ntm.py MIT License | 5 votes |
|
def reset_states(self):
assert self.stateful, 'Layer must be stateful.'
input_shape = self.input_spec[0].shape
if not input_shape[0]:
raise Exception('If a RNN is stateful, a complete ' +
'input_shape must be provided (including batch size).')
if hasattr(self, 'states'):
K.set_value(self.states[0],
np.zeros((input_shape[0], self.output_dim)))
K.set_value(self.states[1],
np.zeros((input_shape[0], self.output_dim)))
else:
self.states = [K.zeros((input_shape[0], self.output_dim)),
K.zeros((input_shape[0], self.output_dim))]
Example 38
| Project: NTM-Keras Author: SigmaQuan File: memory.py MIT License | 5 votes |
|
def initial(number_of_memory_locations, memory_vector_size):
return K.zeros((number_of_memory_locations, memory_vector_size))
Example 39
| Project: keras-optimizers Author: hi-im-ryanli File: frankenstein.py MIT License | 5 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = K.cast(self.iterations, K.floatx()) + 1
# Due to the recommendations in [2], i.e. warming momentum schedule
momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
m_schedule_new = self.m_schedule * momentum_cache_t
m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
self.updates.append((self.m_schedule, m_schedule_new))
shapes = [K.int_shape(p) for p in params]
ms = [K.zeros(shape) for shape in shapes]
vs = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
# the following equations given in [1]
g_prime = g / (1. - m_schedule_new)
m_t = self.beta_1 * m + (1. - self.beta_1) * g
m_t_prime = m_t / (1. - m_schedule_next)
v_t = K.maximum(self.beta_2 * v, K.abs(g))
v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 40
| Project: CapsNet-Collections Author: Jiankai-Sun File: capsulelayers.py GNU General Public License v3.0 | 5 votes |
|
def __init__(self, num_capsule, dim_vector, num_routing=3,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
**kwargs):
super(CapsuleLayer, self).__init__(**kwargs)
self.num_capsule = num_capsule
self.dim_vector = dim_vector
self.num_routing = num_routing
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
Example 41
| Project: conv_qsar_fast Author: connorcoley File: GraphEmbedding_sumAfter.py MIT License | 5 votes |
|
def build(self, input_shape): '''Builds internal weights and paramer attribute''' # NOTE: NEED TO TILE AND EVALUATE SO THAT PARAMS CAN BE VARIABLES # OTHERWISE K.GET_VALUE() DOES NOT WORK # Define template weights for inner FxF W_inner = self.init_inner((self.inner_dim, self.inner_dim)) b_inner = K.zeros((1, self.inner_dim)) # Initialize weights tensor self.W_inner = K.variable(T.tile(W_inner, (self.depth + 1, 1, 1)).eval() + \ initializations.uniform((self.depth + 1, self.inner_dim, self.inner_dim)).eval()) self.W_inner.name = 'T:W_inner' self.b_inner = K.variable(T.tile(b_inner, (self.depth + 1, 1, 1)).eval() + \ initializations.uniform((self.depth + 1, 1, self.inner_dim)).eval()) self.b_inner.name = 'T:b_inner' # # Concatenate third dimension (depth) so different layers can have # # different weights. Now, self.W_inner[#,:,:] corresponds to the # # weight matrix for layer/depth #. # Define template weights for output FxL W_output = self.init_output((self.inner_dim, self.output_dim), scale = self.scale_output) b_output = K.zeros((1, self.output_dim)) # Initialize weights tensor self.W_output = K.variable(T.tile(W_output, (self.depth + 1, 1, 1)).eval()) self.W_output.name = 'T:W_output' self.b_output = K.variable(T.tile(b_output, (self.depth + 1, 1, 1)).eval()) self.b_output.name = 'T:b_output' # # Concatenate third dimension (depth) so different layers can have # # different weights. Now, self.W_output[#,:,:] corresponds to the # # weight matrix for layer/depth #. # Pack params self.trainable_weights = [self.W_inner, self.b_inner, self.W_output, self.b_output] self.params = [self.W_inner, self.b_inner, self.W_output, self.b_output]
Example 42
| Project: conv_qsar_fast Author: connorcoley File: GraphEmbedding.py MIT License | 5 votes |
|
def build(self, input_shape): '''Builds internal weights and paramer attribute''' # NOTE: NEED TO TILE AND EVALUATE SO THAT PARAMS CAN BE VARIABLES # OTHERWISE K.GET_VALUE() DOES NOT WORK # Define template weights for inner FxF W_inner = self.init_inner((self.inner_dim, self.inner_dim)) b_inner = K.zeros((1, self.inner_dim)) # Initialize weights tensor self.W_inner = K.variable(T.tile(W_inner, (self.depth + 1, 1, 1)).eval() + \ initializations.uniform((self.depth + 1, self.inner_dim, self.inner_dim)).eval()) self.W_inner.name = 'T:W_inner' self.b_inner = K.variable(T.tile(b_inner, (self.depth + 1, 1, 1)).eval() + \ initializations.uniform((self.depth + 1, 1, self.inner_dim)).eval()) self.b_inner.name = 'T:b_inner' # # Concatenate third dimension (depth) so different layers can have # # different weights. Now, self.W_inner[#,:,:] corresponds to the # # weight matrix for layer/depth #. # Define template weights for output FxL W_output = self.init_output((self.inner_dim, self.output_dim), scale = self.scale_output) b_output = K.zeros((1, self.output_dim)) # Initialize weights tensor self.W_output = K.variable(T.tile(W_output, (self.depth + 1, 1, 1)).eval()) self.W_output.name = 'T:W_output' self.b_output = K.variable(T.tile(b_output, (self.depth + 1, 1, 1)).eval()) self.b_output.name = 'T:b_output' # # Concatenate third dimension (depth) so different layers can have # # different weights. Now, self.W_output[#,:,:] corresponds to the # # weight matrix for layer/depth #. # Pack params self.trainable_weights = [self.W_inner, self.b_inner, self.W_output, self.b_output] self.params = [self.W_inner, self.b_inner, self.W_output, self.b_output]
Example 43
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 5 votes |
|
def __init__(
self, nb_motifs, motif_len,
init='glorot_uniform',
**kwargs):
self.nb_motifs = nb_motifs
self.motif_len = motif_len
self.input = K.placeholder(ndim=4)
self.init = lambda x: (
initializations.get(init)(x),
K.zeros((self.nb_motifs,))
)
super(ConvolutionDNAShapeBinding, self).__init__(**kwargs)
Example 44
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 5 votes |
|
def __init__(
self,
nb_motifs,
motif_len,
use_three_base_encoding=True,
init='glorot_uniform',
**kwargs):
self.nb_motifs = nb_motifs
self.motif_len = motif_len
self.input = K.placeholder(ndim=4)
self.use_three_base_encoding = use_three_base_encoding
self.kwargs = kwargs
self.W = None
self.b = None
#if isinstance(init, ConvolutionalDNABindingModel):
# self.init = lambda x: (
# K.variable(-init.ddg_array[None,None,:,:]),
# K.variable(np.array([-init.ref_energy,])[:,None])
# )
#else:
# self.init = lambda x: (
# initializations.get(init)(x),
# K.zeros((self.nb_motifs,))
# )
self.init = initializations.get(init)
super(ConvolutionDNASequenceBinding, self).__init__(**kwargs)
Example 45
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 5 votes |
|
def init_filters(self):
if self.use_three_base_encoding:
self.W_shape = (
self.nb_motifs, 3, 1, self.motif_len)
else:
self.W_shape = (
self.nb_motifs, 4, 1, self.motif_len)
self.W = self.init(self.W_shape)
self.b = K.zeros((self.nb_motifs,))
return
Example 46
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 5 votes |
|
def extract_binding_models(self):
mos = []
for i in xrange(self.nb_motifs):
ddg_array = self.W[0].get_value()[i,:,0,:]
ddg_array = np.vstack(
(np.zeros((1, ddg_array.shape[1])), ddg_array)).T
ref_energy = self.b[0].get_value()[i]
mos.append(EnergeticDNABindingModel(-ref_energy, -ddg_array))
return mos
Example 47
| Project: onnx-keras Author: leodestiny File: backend.py MIT License | 5 votes |
|
def _pad(cls, x, pad, dim):
import copy
for i in range(dim):
if pad[i] == 0 and pad[i + dim] == 0:
continue
shape = K.int_shape(x)
zero_shape = copy.deepcopy(list(shape))
zero_shape[i] = pad[i]
zeros1 = K.zeros(zero_shape)
zero_shape[i] = pad[i + dim]
zeros2 = K.zeros(zero_shape)
x = K.concatenate([zeros1, x, zeros2], axis=i)
return x
Example 48
| Project: stochastic_depth_keras Author: dblN File: train.py MIT License | 5 votes |
|
def residual_drop(x, input_shape, output_shape, strides=(1, 1)):
global add_tables
nb_filter = output_shape[0]
conv = Convolution2D(nb_filter, 3, 3, subsample=strides,
border_mode="same", W_regularizer=l2(weight_decay))(x)
conv = BatchNormalization(axis=1)(conv)
conv = Activation("relu")(conv)
conv = Convolution2D(nb_filter, 3, 3,
border_mode="same", W_regularizer=l2(weight_decay))(conv)
conv = BatchNormalization(axis=1)(conv)
if strides[0] >= 2:
x = AveragePooling2D(strides)(x)
if (output_shape[0] - input_shape[0]) > 0:
pad_shape = (1,
output_shape[0] - input_shape[0],
output_shape[1],
output_shape[2])
padding = K.zeros(pad_shape)
padding = K.repeat_elements(padding, K.shape(x)[0], axis=0)
x = Lambda(lambda y: K.concatenate([y, padding], axis=1),
output_shape=output_shape)(x)
_death_rate = K.variable(death_rate)
scale = K.ones_like(conv) - _death_rate
conv = Lambda(lambda c: K.in_test_phase(scale * c, c),
output_shape=output_shape)(conv)
out = merge([conv, x], mode="sum")
out = Activation("relu")(out)
gate = K.variable(1, dtype="uint8")
add_tables += [{"death_rate": _death_rate, "gate": gate}]
return Lambda(lambda tensors: K.switch(gate, tensors[0], tensors[1]),
output_shape=output_shape)([out, x])
Example 49
| Project: CDSGD Author: SCSLabISU File: FASGD.py BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def update_mean_parameters(agentmodels,n_agents):
parameters=[0 for nb in range(n_agents)]
for nb in range(n_agents):
parameters[nb]=agentmodels[nb]._collected_trainable_weights
info_shapes = [K.get_variable_shape(p) for p in agentmodels[0]._collected_trainable_weights]
parameter_mean = [K.zeros(info_shape) for info_shape in info_shapes]
for i in range(len(parameter_mean)):
for nb in range(n_agents):
parameter_mean[i]+=(1/n_agents)*parameters[nb][i]
for nb in range(n_agents):
agentmodels[nb]._collected_trainable_weights=parameter_mean
return agentmodels
Example 50
| Project: dynamic_memory_networks_with_keras Author: vchudinov File: attention_cells.py GNU General Public License v3.0 | 5 votes |
|
def call(self, input_list, initial_state=None, mask=None, training=None):
inputs = input_list
self._generate_dropout_mask(inputs, training=training)
self._generate_recurrent_dropout_mask(inputs, training=training)
# if has_arg(self.layer.call, 'training'):
self.training = training
uses_learning_phase = False
# initial_state = self.get_initial_state(inputs)
last_output, outputs, _ = K.rnn(self.step,
inputs=inputs,
constants=[],
initial_states=[K.zeros(shape= [self.batch_size, self.units])],
#input_length=input_shape,
unroll=False)
if self.return_sequences:
y = outputs
else:
y = last_output
if (hasattr(self, 'activity_regularizer') and
self.activity_regularizer is not None):
regularization_loss = self.activity_regularizer(y)
self.add_loss(regularization_loss, inputs)
if uses_learning_phase:
y._uses_learning_phase = True
if self.return_sequences:
timesteps = input_shape[1]
new_time_steps = list(y.get_shape())
new_time_steps[1] = timesteps
y.set_shape(new_time_steps)
return y
Example 51
| Project: Keras-TextClassification Author: yongzhuo File: keras_radam.py MIT License | 5 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
beta_1_t = K.pow(self.beta_1, t)
beta_2_t = K.pow(self.beta_2, t)
rho = 2 / (1 - self.beta_2) - 1
rho_t = rho - 2 * t * beta_2_t / (1 - beta_2_t)
r_t = K.sqrt(
K.relu(rho_t - 4) * K.relu(rho_t - 2) * rho / ((rho - 4) * (rho - 2) * rho_t)
)
flag = K.cast(rho_t > 4, K.floatx())
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
mhat_t = m_t / (1 - beta_1_t)
vhat_t = K.sqrt(v_t / (1 - beta_2_t))
p_t = p - lr * mhat_t * (flag * r_t / (vhat_t + self.epsilon) + (1 - flag))
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 52
| Project: Implementation-CVPR2015-CNN-for-ReID Author: Ning-Ding File: model_for_market1501.py MIT License | 5 votes |
|
def next(self):
with self.lock:
index_array, current_index, current_batch_size = next(self.index_generator)
batch_x1 = np.zeros(tuple([current_batch_size * 2] + [128,64,3]))
batch_x2 = np.zeros(tuple([current_batch_size * 2] + [128,64,3]))
batch_y = np.zeros([current_batch_size * 2, 2])
for i, j in enumerate(index_array):
x1 = np.array(Image.open(self.folder_dir + self.f[self.train_or_validation][j,0])) / 255.
x2 = np.array(Image.open(self.folder_dir + self.f[self.train_or_validation][j,1])) / 255.
if np.random.rand() > self.flag:
x1 = self.image_data_generator.random_transform(x1.astype('float32'))
if np.random.rand() > self.flag:
x2 = self.image_data_generator.random_transform(x2.astype('float32'))
batch_x1[2*i] = x1
batch_x2[2*i] = x2
batch_y[2*i][1] = 1
while True:
index_1,index_2 = np.random.choice(self.path_list,2)
if index_1[6] != index_2[6] and index_1[0:4] != index_2[0:4]:
break
x1 = np.array(Image.open(self.folder_dir + index_1)) / 255.
x2 = np.array(Image.open(self.folder_dir + index_2)) / 255.
batch_x1[2*i+1] = x1
batch_x2[2*i+1] = x2
batch_y[2*i+1][0] = 1
return [batch_x1,batch_x2], batch_y
Example 53
| Project: deep-models Author: LaurentMazare File: rhn.py Apache License 2.0 | 4 votes |
|
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.input_dim = input_shape[2]
if self.stateful:
self.reset_states()
else:
# initial states: all-zero tensor of shape (output_dim)
self.states = [None]
self.W_t = self.init((self.input_dim, self.output_dim),
name='{}_W_t'.format(self.name))
self.b_t = K.zeros((self.output_dim,), name='{}_b_t'.format(self.name))
self.W_h = self.init((self.input_dim, self.output_dim),
name='{}_W_h'.format(self.name))
self.b_h = K.zeros((self.output_dim,), name='{}_b_h'.format(self.name))
self.U_ts, self.b_ts = [], []
self.U_hs, self.b_hs = [], []
for l in xrange(self.L):
self.U_ts.append(self.inner_init((self.output_dim, self.output_dim), name='{}_U_t{}'.format(self.name, l)))
self.b_ts.append(K.zeros((self.output_dim,), name='{}_b_t{}'.format(self.name, l)))
self.U_hs.append(self.inner_init((self.output_dim, self.output_dim), name='{}_U_h{}'.format(self.name, l)))
self.b_hs.append(K.zeros((self.output_dim,), name='{}_b_h{}'.format(self.name, l)))
self.trainable_weights = [ self.W_t, self.b_t, self.W_h, self.b_h] + self.U_ts + self.U_hs + self.b_ts + self.b_hs
self.W = K.concatenate([self.W_t, self.W_h])
self.U = K.concatenate(self.U_ts + self.U_hs)
self.b = K.concatenate([self.b_t, self.b_h] + self.b_ts + self.b_hs)
self.regularizers = []
if self.W_regularizer:
self.W_regularizer.set_param(self.W)
self.regularizers.append(self.W_regularizer)
if self.U_regularizer:
self.U_regularizer.set_param(self.U)
self.regularizers.append(self.U_regularizer)
if self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
Example 54
| Project: DeepLearningMugenKnock Author: yoyoyo-yo File: cgan_cifar10_keras.py MIT License | 4 votes |
|
def G_model():
inputs = Input([100, ], name="x")
con_x = Input([num_classes, ], name="con_x")
con_x2 = Input([img_height, img_width, num_classes], name="con_x2")
#con_x = K.zeros([None, num_classes, 1, 1])
#print(con_x.shape)
#con_x = np.zeros([len(_con_x), num_classes, 1, 1], dtype=np.float32)
#con_x[np.arange(len(_con_x)), _con_x] = 1
x = concatenate([inputs, con_x], axis=-1)
in_h = int(img_height / 16)
in_w = int(img_width / 16)
d_dim = 256
base = 128
x = Dense(in_h * in_w * d_dim, name='g_dense1',
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Reshape((in_h, in_w, d_dim), input_shape=(d_dim * in_h * in_w,))(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_dense1_bn')(x)
# 1/8
x = Conv2DTranspose(base*4, (5, 5), name='g_conv1', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv1_bn')(x)
# 1/4
x = Conv2DTranspose(base*2, (5, 5), name='g_conv2', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv2_bn')(x)
# 1/2
x = Conv2DTranspose(base, (5, 5), name='g_conv3', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv3_bn')(x)
# 1/1
x = Conv2DTranspose(channel, (5, 5), name='g_out', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), bias_initializer=Constant())(x)
x = Activation('tanh')(x)
#con_x = np.zerns([len(_con_x), num_classes, img_height, img_width], dtype=np.float32)
#con_x[np.arange(len(_con_x)), _con_x] = 1
x2 = concatenate([x, con_x2], axis=-1)
model = Model(inputs=[inputs, con_x, con_x2], outputs=[x], name='G')
gan_g_model = Model(inputs=[inputs, con_x, con_x2], outputs=[x2], name='GAN_G')
return model, gan_g_model
Example 55
| Project: DeepLearningMugenKnock Author: yoyoyo-yo File: cgan_mnist_keras.py MIT License | 4 votes |
|
def G_model():
inputs = Input([100, ], name="x")
con_x = Input([num_classes, ], name="con_x")
con_x2 = Input([img_height, img_width, num_classes], name="con_x2")
#con_x = K.zeros([None, num_classes, 1, 1])
#print(con_x.shape)
#con_x = np.zeros([len(_con_x), num_classes, 1, 1], dtype=np.float32)
#con_x[np.arange(len(_con_x)), _con_x] = 1
x = concatenate([inputs, con_x], axis=-1)
in_h = int(img_height / 4)
in_w = int(img_width / 4)
d_dim = 256
base = 128
x = Dense(in_h * in_w * d_dim, name='g_dense1',
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Reshape((in_h, in_w, d_dim), input_shape=(d_dim * in_h * in_w,))(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_dense1_bn')(x)
# 1/8
#x = Conv2DTranspose(base*4, (5, 5), name='g_conv1', padding='same', strides=(2,2),
# kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
#x = Activation('relu')(x)
#x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv1_bn')(x)
# 1/4
#x = Conv2DTranspose(base*2, (5, 5), name='g_conv2', padding='same', strides=(2,2),
# kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
#x = Activation('relu')(x)
#x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv2_bn')(x)
# 1/2
x = Conv2DTranspose(base, (5, 5), name='g_conv3', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), use_bias=False)(x)
x = Activation('relu')(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5, name='g_conv3_bn')(x)
# 1/1
x = Conv2DTranspose(channel, (5, 5), name='g_out', padding='same', strides=(2,2),
kernel_initializer=RN(mean=0.0, stddev=0.02), bias_initializer=Constant())(x)
x = Activation('tanh')(x)
#con_x = np.zerns([len(_con_x), num_classes, img_height, img_width], dtype=np.float32)
#con_x[np.arange(len(_con_x)), _con_x] = 1
x2 = concatenate([x, con_x2], axis=-1)
model = Model(inputs=[inputs, con_x], outputs=[x], name='G')
gan_g_model = Model(inputs=[inputs, con_x, con_x2], outputs=[x2], name='GAN_G')
return model, gan_g_model
Example 56
| Project: bert_lamb_pretrain Author: goldenbili File: keras_lamb.py Apache License 2.0 | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
m_t_hat = m_t / (1. - K.pow(self.beta_1, t))
v_t_hat = v_t / (1. - K.pow(self.beta_2, t))
p_dash = m_t_hat / (K.sqrt(v_t_hat + self.epsilon))
if self.weight_decay > 0.:
wd = self.weight_decay * p
p_dash = p_dash + wd
r1 = K.sqrt(K.sum(K.square(p)))
r2 = K.sqrt(K.sum(K.square(p_dash)))
r = tf.where(tf.greater(r1, 0.),
tf.where(tf.greater(r2, 0.),
r1 / r2,
1.0),
1.0)
# r = r1 / r2
eta = r * lr
p_t = p - eta * p_dash
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 57
| Project: keras_radam Author: forin-xyz File: radam.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay:
lr = lr * (1. / (1. + self.decay * K.cast(
self.iterations, K.dtype(self.decay)
)))
t = K.cast(self.iterations, K.floatx()) + 1.
beta_1 = self.beta_1
beta_2 = self.beta_2
beta_1_t = K.pow(beta_1, t)
beta_2_t = K.pow(beta_2, t)
rho_inf = 2. / (1. - beta_2) - 1.
rho_t = rho_inf - 2. * t * beta_2_t / (1. - beta_2_t)
r_t = K.sqrt(
K.relu(rho_t - 4.) * (rho_t - 2.) * rho_inf / (
(rho_inf - 4.) * (rho_inf - 2.) * rho_t )
)
flag = K.cast(rho_t > 4., K.floatx())
ms = [K.zeros(K.int_shape(p)) for p in params]
vs = [K.zeros(K.int_shape(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t = beta_1 * m + (1. - beta_1) * g
v_t = beta_2 * v + (1. - beta_2) * K.square(g)
m_hat_t = m_t / (1. - beta_1_t)
v_hat_t = K.sqrt(v_t / (1. - beta_2_t))
new_p = p - lr * (r_t / (v_hat_t + self.epsilon) + flag - 1.)* m_hat_t
if getattr(p, "constraint", None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
return self.updates
Example 58
| Project: CapsNet-Fashion-MNIST Author: subarnop File: capsulelayers.py GNU General Public License v3.0 | 4 votes |
|
def call(self, inputs, training=None):
# inputs.shape=[None, input_num_capsule, input_dim_vector]
# Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector]
inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2)
# Replicate num_capsule dimension to prepare being multiplied by W
# Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector]
inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1])
"""
# Begin: inputs_hat computation V1 ---------------------------------------------------------------------#
# Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size.
# w_tiled.shape = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector]
w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1])
# Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3])
# End: inputs_hat computation V1 ---------------------------------------------------------------------#
"""
# Begin: inputs_hat computation V2 ---------------------------------------------------------------------#
# Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow.
# inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]),
elems=inputs_tiled,
initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector]))
# End: inputs_hat computation V2 ---------------------------------------------------------------------#
"""
# Begin: routing algorithm V1, dynamic ------------------------------------------------------------#
def body(i, b, outputs):
c = tf.nn.softmax(b, dim=2) # dim=2 is the num_capsule dimension
outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
if i != 1:
b = b + K.sum(inputs_hat * outputs, -1, keepdims=True)
return [i-1, b, outputs]
cond = lambda i, b, inputs_hat: i > 0
loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)]
shape_invariants = [tf.TensorShape([]),
tf.TensorShape([None, self.input_num_capsule, self.num_capsule, 1, 1]),
tf.TensorShape([None, 1, self.num_capsule, 1, self.dim_vector])]
_, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants)
# End: routing algorithm V1, dynamic ------------------------------------------------------------#
"""
# Begin: routing algorithm V2, static -----------------------------------------------------------#
# Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance
assert self.num_routing > 0, 'The num_routing should be > 0.'
for i in range(self.num_routing):
c = tf.nn.softmax(self.bias, dim=2) # dim=2 is the num_capsule dimension
# outputs.shape=[None, 1, num_capsule, 1, dim_vector]
outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
# last iteration needs not compute bias which will not be passed to the graph any more anyway.
if i != self.num_routing - 1:
# self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True))
self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True)
# tf.summary.histogram('BigBee', self.bias) # for debugging
# End: routing algorithm V2, static ------------------------------------------------------------#
return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector])
Example 59
| Project: keras-adamw Author: OverLordGoldDragon File: optimizers.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
self.updates.append(K.update_add(self.t_cur, 1))
lr = self.learning_rate
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
# momentum
shapes = [K.int_shape(p) for p in params]
moments = [K.zeros(shape, name='moment_' + str(i))
for (i, shape) in enumerate(shapes)]
self.weights = [self.iterations] + moments
total_iterations = self.total_iterations
# Cosine annealing
if self.use_cosine_annealing and total_iterations != 0:
self.eta_t = _compute_eta_t(self)
self.lr_t = lr * self.eta_t # for external tracking
for p, g, m in zip(params, grads, moments):
# Learning rate multipliers
lr_t = self.learning_rate
if self.lr_multipliers is not None:
lr_t = _apply_lr_multiplier(self, lr_t, p)
v = self.momentum * m - self.eta_t * lr_t * g # velocity
self.updates.append(K.update(m, v))
if self.nesterov:
p_t = p + self.momentum * v - self.eta_t * lr_t * g
else:
p_t = p + v
# Weight decays
if p.name in self.weight_decays.keys() and total_iterations != 0:
p_t = _apply_weight_decays(self, p, p_t)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self._init_notified = True
return self.updates
Example 60
| Project: keras-adamw Author: OverLordGoldDragon File: optimizers_225.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
self.updates.append(K.update_add(self.t_cur, 1))
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
total_iterations = self.total_iterations
# Cosine annealing
if self.use_cosine_annealing and total_iterations != 0:
self.eta_t = _compute_eta_t(self)
self.lr_t = lr_t * self.eta_t # for external tracking
lr_t_premult = lr_t
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
# Learning rate multipliers
if self.lr_multipliers is not None:
lr_t = _apply_lr_multiplier(self, lr_t_premult, p)
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
# Weight decays
if p.name in self.weight_decays.keys() and total_iterations != 0:
p_t = _apply_weight_decays(self, p, p_t)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self._init_notified = True
return self.updates
Example 61
| Project: keras-adamw Author: OverLordGoldDragon File: optimizers_225.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
self.updates.append(K.update_add(self.t_cur, 1))
t = K.cast(self.iterations, K.floatx()) + 1
# Due to the recommendations in [2], i.e. warming momentum schedule
momentum_cache_t = self.beta_1 * (1. - 0.5 * (
K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (
K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
m_schedule_new = self.m_schedule * momentum_cache_t
m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
self.updates.append((self.m_schedule, m_schedule_new))
shapes = [K.int_shape(p) for p in params]
ms = [K.zeros(shape) for shape in shapes]
vs = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + ms + vs
total_iterations = self.total_iterations
# Cosine annealing
if self.use_cosine_annealing and total_iterations != 0:
self.eta_t = _compute_eta_t(self)
self.lr_t = self.lr * self.eta_t # for external tracking
for p, g, m, v in zip(params, grads, ms, vs):
# Learning rate multipliers
lr_t = self.lr
if self.lr_multipliers is not None:
lr_t = _apply_lr_multiplier(self, lr_t, p)
# the following equations given in [1]
g_prime = g / (1. - m_schedule_new)
m_t = self.beta_1 * m + (1. - self.beta_1) * g
m_t_prime = m_t / (1. - m_schedule_next)
v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
m_t_bar = (1. - momentum_cache_t) * g_prime + (
momentum_cache_t_1 * m_t_prime)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
p_t = p - self.eta_t * lr_t * m_t_bar / (
K.sqrt(v_t_prime) + self.epsilon)
# Weight decays
if p.name in self.weight_decays.keys() and total_iterations != 0:
p_t = _apply_weight_decays(self, p, p_t)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self._init_notified = True
return self.updates
Example 62
| Project: keras-adamw Author: OverLordGoldDragon File: optimizers_225.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
self.updates.append(K.update_add(self.t_cur, 1))
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
# momentum
shapes = [K.int_shape(p) for p in params]
moments = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + moments
total_iterations = self.total_iterations
# Cosine annealing
if self.use_cosine_annealing and total_iterations != 0:
self.eta_t = _compute_eta_t(self)
self.lr_t = lr * self.eta_t # for external tracking
for p, g, m in zip(params, grads, moments):
# Learning rate multipliers
lr_t = self.lr
if self.lr_multipliers is not None:
lr_t = _apply_lr_multiplier(self, lr_t, p)
v = self.momentum * m - self.eta_t * lr_t * g # velocity
self.updates.append(K.update(m, v))
if self.nesterov:
p_t = p + self.momentum * v - self.eta_t * lr_t * g
else:
p_t = p + v
# Weight decays
if p.name in self.weight_decays.keys() and total_iterations != 0:
p_t = _apply_weight_decays(self, p, p_t)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
self._init_notified = True
return self.updates
Example 63
| Project: dockerizeme Author: dockerizeme File: snippet.py Apache License 2.0 | 4 votes |
|
def call(self, x, mask=None):
# input shape: (nb_samples, time (padded with zeros), input_dim)
input_shape = self.input_spec[0].shape
if K._BACKEND == 'tensorflow':
if not input_shape[1]:
raise Exception('When using TensorFlow, you should define '
'explicitly the number of timesteps of '
'your sequences.\n'
'If your first layer is an Embedding, '
'make sure to pass it an "input_length" '
'argument. Otherwise, make sure '
'the first layer has '
'an "input_shape" or "batch_input_shape" '
'argument, including the time axis. '
'Found input shape at layer ' + self.name +
': ' + str(input_shape))
if self.layer.stateful:
initial_states = self.layer.states
else:
initial_states = self.layer.get_initial_states(x)
constants = self.get_constants(x)
preprocessed_input = self.layer.preprocess_input(x)
last_output, outputs, states = K.rnn(self.step, preprocessed_input,
initial_states,
go_backwards=self.layer.go_backwards,
mask=mask,
constants=constants,
unroll=self.layer.unroll,
input_length=input_shape[1])
if self.layer.stateful:
self.updates = []
for i in range(len(states)):
self.updates.append((self.layer.states[i], states[i]))
if self.layer.return_sequences:
return outputs
else:
return last_output
# test likes in https://github.com/fchollet/keras/blob/master/tests/keras/layers/test_wrappers.py
Example 64
| Project: keras_extension Author: k1414st File: optimizers.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
eta_l_t = self.terminal_bound - \
(self.terminal_bound - self.lower_bound) / \
((1. - self.beta_2) * t + 1)
eta_u_t = self.terminal_bound + \
(self.upper_bound - self.terminal_bound) / \
((1. - self.beta_2) * t)
clipped_lr_t = K.minimum(
K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
p_t = p - clipped_lr_t * m_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 65
| Project: keras_extension Author: k1414st File: optimizers.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
eta_l_t = self.terminal_bound - \
(self.terminal_bound - self.lower_bound) / \
((1. - self.beta_2) * t + 1)
eta_u_t = self.terminal_bound + \
(self.upper_bound - self.terminal_bound) / \
((1. - self.beta_2) * t)
clipped_lr_t = K.minimum(
K.maximum(lr_t / (K.sqrt(v_t) + self.epsilon), eta_l_t), eta_u_t)
p_t = p - clipped_lr_t * m_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 66
| Project: Coloring-greyscale-images Author: emilwallner File: AdamAccumulate.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
# self.iterations incremented after processing a batch
# batch: 1 2 3 4 5 6 7 8 9
# self.iterations: 0 1 2 3 4 5 6 7 8
# update_switch = 1: x x (if accum_iters=4)
update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)
update_switch = K.cast(update_switch, K.floatx())
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))
self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))
self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))
return self.updates
Example 67
| Project: NTM-Keras Author: SigmaQuan File: ntm.py MIT License | 4 votes |
|
def reset_states(self):
print("begin reset_states(self)")
assert self.stateful, 'Layer must be stateful.'
input_shape = self.input_spec[0].shape
self.depth = 0
if not input_shape[0]:
raise Exception('If a RNN is stateful, a complete ' +
'input_shape must be provided (including batch size).')
if hasattr(self, 'states'):
# K.set_value(self.states[0],
# np.zeros((input_shape[0], self.output_dim)))
# K.set_value(self.states[1],
# np.zeros((input_shape[0], self.output_dim)))
# add by Robot Steven ****************************************#
# previous inner memory
K.set_value(self.states[0],
np.zeros((input_shape[0], self.controller_output_dim)))
# previous inner cell
K.set_value(self.states[1],
np.zeros((input_shape[0], self.controller_output_dim)))
# previous memory
K.set_value(self.states[2],
np.zeros((input_shape[0], self.memory_dim * self.memory_size)))
# K.set_value(self.states[2],
# np.zeros((input_shape[0], self.memory_size, self.memory_dim)))
# previous writing addresses
K.set_value(self.states[3],
np.zeros((input_shape[0], self.num_write_head * self.memory_size)))
# K.set_value(self.states[3],
# np.zeros((input_shape[0], self.num_write_head * self.memory_size)))
# previous reading addresses
K.set_value(self.states[4],
np.zeros((input_shape[0], self.num_read_head * self.memory_size)))
# previous reading content
K.set_value(self.states[5],
np.zeros((input_shape[0], self.num_read_head * self.memory_dim)))
# add by Robot Steven ****************************************#
else:
# self.states = [K.zeros((input_shape[0], self.output_dim)),
# K.zeros((input_shape[0], self.output_dim))]
# add by Robot Steven ****************************************#
self.states = [K.zeros((input_shape[0], self.controller_output_dim)), # h_tm1
K.zeros((input_shape[0], self.controller_output_dim)), # c_tm1]
K.zeros((input_shape[0], self.memory_dim * self.memory_size)),
# K.zeros((input_shape[0], self.memory_size, self.memory_dim)),
K.zeros((input_shape[0], self.num_write_head * self.memory_size)),
K.zeros((input_shape[0], self.num_read_head * self.memory_size)),
K.zeros((input_shape[0], self.num_read_head * self.memory_dim))]
# add by Robot Steven ****************************************#
print("end reset_states(self)\n")
Example 68
| Project: keras-optimizers Author: hi-im-ryanli File: frankenstein.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = K.cast(self.iterations, K.floatx()) + 1
# Due to the recommendations in [2], i.e. warming momentum schedule
momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
m_schedule_new = self.m_schedule * momentum_cache_t
m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
self.updates.append((self.m_schedule, m_schedule_new))
shapes = [K.int_shape(p) for p in params]
ms = [K.zeros(shape) for shape in shapes]
vs = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
# the following equations given in [1]
g_prime = g / (1. - m_schedule_new)
m_t = self.beta_1 * m + (1. - self.beta_1) * g
m_t_prime = m_t / (1. - m_schedule_next)
if np.random.choice([1, -1]) == 1:
v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
else:
v_t = K.maximum(self.beta_2 * v, K.abs(g))
v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 69
| Project: keras-optimizers Author: hi-im-ryanli File: frankenstein.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
shapes = [K.int_shape(p) for p in params]
accumulators = [K.zeros(shape) for shape in shapes]
delta_accumulators = [K.zeros(shape) for shape in shapes]
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr *= (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v, a, d_a in zip(params, grads, ms, vs, accumulators, delta_accumulators):
# update accumulator
new_a = self.rho * a + (1. - self.rho) * K.square(g)
self.updates.append(K.update(a, new_a))
# use the new accumulator and the *old* delta_accumulator
update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)
m_t = (self.beta_1 * m) + (1. - self.beta_1) * update
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(update)
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 70
| Project: CapsNet-Collections Author: Jiankai-Sun File: capsulelayers.py GNU General Public License v3.0 | 4 votes |
|
def call(self, inputs, training=None):
# inputs.shape=[None, input_num_capsule, input_dim_vector]
# Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector]
inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2)
# Replicate num_capsule dimension to prepare being multiplied by W
# Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector]
inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1])
"""
# Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size.
# Now W has shape = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector]
w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1])
# Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3])
"""
# Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow.
# inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector]
inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]),
elems=inputs_tiled,
initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector]))
"""
# Routing algorithm V1. Use tf.while_loop in a dynamic way.
def body(i, b, outputs):
c = tf.nn.softmax(self.bias, dim=2) # dim=2 is the num_capsule dimension
outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
b = b + K.sum(inputs_hat * outputs, -1, keepdims=True)
return [i-1, b, outputs]
cond = lambda i, b, inputs_hat: i > 0
loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)]
_, _, outputs = tf.while_loop(cond, body, loop_vars)
"""
# Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance
assert self.num_routing > 0, 'The num_routing should be > 0.'
for i in range(self.num_routing):
c = tf.nn.softmax(self.bias, dim=2) # dim=2 is the num_capsule dimension
# outputs.shape=[None, 1, num_capsule, 1, dim_vector]
outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True))
# last iteration needs not compute bias which will not be passed to the graph any more anyway.
if i != self.num_routing - 1:
# self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True))
self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True)
# tf.summary.histogram('BigBee', self.bias) # for debugging
return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector])
Example 71
| Project: conv_qsar_fast Author: connorcoley File: GraphEmbedding_sumAfter.py MIT License | 4 votes |
|
def get_output_singlesample(self, original_graph): '''For a 3D tensor, get the output. Avoids the need for even more complicated vectorization''' # Check padding if self.padding: rowsum = original_graph.sum(axis = 0) # add across trim = rowsum[:, -1] # last feature == bond flag trim_to = T.eq(trim, 0).nonzero()[0][0] # first index with no bonds original_graph = original_graph[:trim_to, :trim_to, :] # reduced graph # Get attribute values for layer zero # where attributes is a 2D tensor and attributes[#, :] is the vector of # concatenated node and edge attributes. In the first layer (depth 0), the # edge attribute section is initialized to zeros. After increasing depth, howevevr, # this part of the vector will become non-zero. # The first attributes matrix is just graph_tensor[i, i, :], but we can't use that # kind of advanced indexing # Want to extract tensor diagonal as matrix, but can't do that directly... # Want to loop over third dimension, so need to dimshuffle (attributes, updates) = theano.scan(lambda x: x.diagonal(), sequences = original_graph.dimshuffle((2, 0, 1))) attributes.name = 'attributes' # Now the attributes is (N_features x N_nodes), so we need to transpose attributes = attributes.T attributes.name = 'attributes post-transpose' # Get initial fingerprint presum_fp = self.attributes_to_fp_contribution(attributes, 0) fp = K.sum(presum_fp, axis = 0) # sum across atom contributions fp.name = 'initial fingerprint' # Get bond matrix bonds = original_graph[:, :, -1] # flag if the bond is present, (N_atom x N_atom) bonds.name = 'bonds' graph = original_graph # Iterate through different depths, updating attributes each time for depth in range(self.depth): (attributes, graph) = self.attributes_update(attributes, depth + 1, graph, original_graph, bonds) presum_fp_new = self.attributes_to_fp_contribution(attributes, depth + 1) presum_fp_new.name = 'presum_fp_new contribution' fp = fp + K.sum(presum_fp_new, axis = 0) return fp
Example 72
| Project: conv_qsar_fast Author: connorcoley File: GraphEmbedding.py MIT License | 4 votes |
|
def get_output_singlesample(self, original_graph): '''For a 3D tensor, get the output. Avoids the need for even more complicated vectorization''' # Check padding if self.padding: rowsum = original_graph.sum(axis = 0) # add across trim = rowsum[:, -1] # last feature == bond flag trim_to = T.eq(trim, 0).nonzero()[0][0] # first index with no bonds original_graph = original_graph[:trim_to, :trim_to, :] # reduced graph # Get attribute values for layer zero # where attributes is a 2D tensor and attributes[#, :] is the vector of # concatenated node and edge attributes. In the first layer (depth 0), the # edge attribute section is initialized to zeros. After increasing depth, howevevr, # this part of the vector will become non-zero. # The first attributes matrix is just graph_tensor[i, i, :], but we can't use that # kind of advanced indexing # Want to extract tensor diagonal as matrix, but can't do that directly... # Want to loop over third dimension, so need to dimshuffle (attributes, updates) = theano.scan(lambda x: x.diagonal(), sequences = original_graph.dimshuffle((2, 0, 1))) attributes.name = 'attributes' # Now the attributes is (N_features x N_nodes), so we need to transpose attributes = attributes.T attributes.name = 'attributes post-transpose' # Get initial fingerprint presum_fp = self.attributes_to_fp_contribution(attributes, 0) fp = K.sum(presum_fp, axis = 0) # sum across atom contributions fp.name = 'initial fingerprint' # Get bond matrix bonds = original_graph[:, :, -1] # flag if the bond is present, (N_atom x N_atom) bonds.name = 'bonds' # Iterate through different depths, updating attributes each time graph = original_graph for depth in range(self.depth): (attributes, graph) = self.attributes_update(attributes, depth + 1, graph, original_graph, bonds) presum_fp_new = self.attributes_to_fp_contribution(attributes, depth + 1) presum_fp_new.name = 'presum_fp_new contribution' fp = fp + K.sum(presum_fp_new, axis = 0) return fp
Example 73
| Project: pyDNAbinding Author: nboley File: keraslib.py GNU General Public License v2.0 | 4 votes |
|
def iter_weighted_batch_samples(model, batch_iterator, oversampling_ratio=1):
"""Iter batches where poorly predicted samples are more frequently selected.
model: the model to predict from
batch_iterator: batch iterator to draw samples from
oversampling_ratio: how many batch_iterator batches to sample from
"""
assert oversampling_ratio > 0
while True:
# group the output of oversampling_ratio batches together
all_batches = defaultdict(list)
for i in xrange(oversampling_ratio):
for key, val in next(batch_iterator).iteritems():
all_batches[key].append(val)
# find the batch size
batch_size = next(all_batches.itervalues())[0].shape[0]
# stack the output
for key, batches in all_batches.iteritems():
all_batches[key] = np.vstack(batches)
# weight the batch values
pred_vals = model.predict_on_batch(all_batches)
weights = np.zeros(batch_size*oversampling_ratio)
for key in pred_vals:
if key.endswith('binding_occupancies'): continue
losses = (pred_vals[key] - all_batches[key])**2
# set ambiguous labels to 0
losses[all_batches[key] < -0.5] = 0
# add to the losses,
inner_weights = losses.sum(1)
# re-weight the rows with ambiguous labels
inner_weights *= losses.shape[1]/((
all_batches[key] > -0.5).sum(1) + 1e-6)
# make sure that every row has some weight
inner_weights += 1e-6
weights += inner_weights
# normalize the weights to 1
weights /= weights.sum()
# downsample batch_size observations
output = {}
loss_indices = np.random.choice(
len(weights), size=batch_size, replace=False, p=weights)
for key in all_batches.keys():
output[key] = all_batches[key][loss_indices,:]
yield output
return
Example 74
| Project: FormicID Author: naturalis File: optimizer.py MIT License | 4 votes |
|
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = self.iterations + 1
shapes = [K.get_variable_shape(p) for p in params]
ms = [K.zeros(shape) for shape in shapes]
vs = [K.zeros(shape) for shape in shapes]
loss_prev = K.variable(0)
self.updates.append(K.update(loss_prev, loss))
# Calculate the numerator of the Eve coefficient
d_num_t = K.abs(loss_prev - loss)
self.updates.append(K.update(self.d_num, d_num_t))
# Calculate the denominator of the Eve coefficient
d_den_t = K.abs(K.minimum(loss_prev, loss) - self.loss_min)
self.updates.append(K.update(self.d_den, d_den_t))
# Calculate the Eve coefficient. At the first iteration, it is 1.
d_tilde_t = K.clip(
(d_num_t + self.fmin_pos) / (d_den_t + self.fmin_pos),
1. / self.c,
self.c,
)
d_t = (self.beta_3 * self.d) + (1. - self.beta_3) * d_tilde_t
d_t = K.switch(K.greater(t, 1), d_t, K.constant(1))
self.updates.append(K.update(self.d, d_t))
# Calculate the effective learning rate as lr / (d * decay)
lr_eff_t = self.lr / (d_t * (1. + (self.iterations * self.decay)))
self.updates.append(K.update(self.lr_eff, lr_eff_t))
# Apply bias correction to the learning rate
lr_hat_t = (
lr_eff_t
* K.sqrt(1. - K.pow(self.beta_2, t))
/ (1. - K.pow(self.beta_1, t))
)
# Update per parameter
for p, g, m, v in zip(params, grads, ms, vs):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
self.updates.append(K.update(m, m_t))
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
self.updates.append(K.update(v, v_t))
p_t = p - lr_hat_t * m_t / (K.sqrt(v_t) + self.epsilon)
new_p = p_t
# Apply constraints
if p in constraints:
c = constraints[p]
new_p = c(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 75
| Project: MCLNN Author: fadymedhat File: layers.py MIT License | 4 votes |
|
def build(self, input_shape):
assert len(input_shape) == 3
input_dim = input_shape[2]
self.input_spec = [InputSpec(dtype=K.floatx(),
shape=(None, input_shape[1], input_dim))]
self.W = self.init((self.order * 2 + 1, input_dim, self.output_dim),
name='{}_W'.format(self.name))
# self.W = self.print_dim('W :',self.W)
self.b = K.zeros((self.output_dim,),
name='{}_b'.format(self.name))
self.trainable_weights = [self.W, self.b]
self.weightmask = self.construct_mask(feature_count=input_dim,
hidden_count=self.output_dim,
bandwidth=self.bandwidth,
overlap=self.overlap,
layer_is_masked=self.layer_is_masked)
# --K_START -- Refer to keras documentation for the below.
self.regularizers = []
if self.W_regularizer:
self.W_regularizer.set_param(self.W)
self.regularizers.append(self.W_regularizer)
if self.b_regularizer:
self.b_regularizer.set_param(self.b)
self.regularizers.append(self.b_regularizer)
if self.activity_regularizer:
self.activity_regularizer.set_layer(self)
self.regularizers.append(self.activity_regularizer)
self.constraints = {}
if self.W_constraint:
self.constraints[self.W] = self.W_constraint
if self.b_constraint:
self.constraints[self.b] = self.b_constraint
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
# --K_START -- Refer to keras documentation for the above.
Example 76
| Project: keras-adabound Author: titu1994 File: adabound.py MIT License | 4 votes |
|
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
# Applies bounds on actual learning rate
step_size = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
final_lr = self.final_lr * lr / self.base_lr
lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.))
upper_bound = final_lr * (1. + 1. / (self.gamma * t))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsbound:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
# apply weight decay
if self.weight_decay != 0.:
g += self.weight_decay * K.stop_gradient(p)
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
if self.amsbound:
vhat_t = K.maximum(vhat, v_t)
denom = (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
denom = (K.sqrt(v_t) + self.epsilon)
# Compute the bounds
step_size_p = step_size * K.ones_like(denom)
step_size_p_bound = step_size_p / denom
bounded_lr_t = m_t * K.minimum(K.maximum(step_size_p_bound,
lower_bound), upper_bound)
p_t = p - bounded_lr_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 77
| Project: CDSGD Author: SCSLabISU File: EASGD.py BSD 3-Clause "New" or "Revised" License | 4 votes |
|
def get_updates(self, params, constraints, loss):
parameter_mean=globals()["parameter_mean"]
epoch_count=globals()["epoch_count"]
copy_shapes = [K.get_variable_shape(p) for p in params]
parameter_copy = [K.zeros(copy_shape) for copy_shape in copy_shapes]
parameter_copy=params
grads = self.get_gradients(loss, parameter_copy)
self.updates = []
lr = self.lr
alpha=self.alpha
communication_period=self.communication_period
if self.initial_decay > 0:
lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
self.updates .append(K.update_add(self.iterations, 1))
# momentum
shapes = [K.get_variable_shape(p) for p in params]
moments = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + moments
temp_param_mean=[]
for p, pi, pm, g, m in zip(params,parameter_copy, parameter_mean, grads, moments):
# for i in range(len(params)):
# p=params[i]
# p1=parameter_copy[i]
# pm=parameter_mean[i]
# g=grads[i]
# m=moments[i]
# Momentum term
#v = self.momentum * m - lr * g # velocity
v = - lr * g # no momentum
self.updates.append(K.update(m, v))
if self.nesterov:
if (epoch_count%communication_period)==0:
new_pm = pm + alpha*(pi-pm)
new_p = p - alpha*(pi-pm) + self.momentum * v - lr * g
temp_param_mean.append(new_p)
else:
new_p = p + self.momentum * v - lr * g
else:
if (epoch_count%communication_period)==0:
new_pm = pm + alpha*(pi-pm)
new_p = p - alpha*(pi-pm) + v
temp_param_mean.append(new_pm)
else:
new_p = p + v
# apply constraints
if p in constraints:
c = constraints[p]
new_p = c(new_p)
self.updates.append(K.update(p, new_p))
parameter_mean=temp_param_mean
return self.updates
Example 78
| Project: CDSGD Author: SCSLabISU File: CDSGD.py BSD 3-Clause "New" or "Revised" License | 4 votes |
|
def get_updates(self, params, constraints, loss):
pi=self.pi
n_agents=self.n_agents
agent_id=self.agent_id
parameters=globals()["parameters"]
info_shapes = [K.get_variable_shape(p) for p in params]
parameter_copy = [0 for nb in range(n_agents)]
for nb in range(n_agents):
parameter_copy[nb]=parameters[nb]
if (nb==agent_id):
parameter_copy[nb]=params
grads = self.get_gradients(loss, params)
self.updates = []
lr = self.lr
if self.initial_decay > 0:
lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
self.updates .append(K.update_add(self.iterations, 1))
# momentum
shapes = [K.get_variable_shape(p) for p in params]
moments = [K.zeros(shape) for shape in shapes]
dist_accumulator = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + moments
#for p, g, m, d, p_agents in zip(params, grads, moments, dist_accumulator, *parameter_copy):
for i in range(len(params)):
p=params[i]
g=grads[i]
m=moments[i]
d=dist_accumulator[i]
# Momentum term
v = self.momentum * m - lr * g # velocity
#v = - lr * g # no momentum
self.updates.append(K.update(m, v))
if self.nesterov:
for nb in range(n_agents):
d+=pi[nb][agent_id]*parameter_copy[nb][i]
new_p = d + self.momentum * v - lr * g
else:
# This is for Debug only
# if count>5:
# raise ValueError('parameters: ' + str(p1) + str(p2) + str(p3) + str(p4) + str(p5) )
for nb in range(n_agents):
d+=pi[nb][agent_id]*parameter_copy[nb][i]
#raise ValueError('pi:' + str(K.eval(pi)))
new_p = d + v
# apply constraints
if p in constraints:
c = constraints[p]
new_p = c(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 79
| Project: CDSGD Author: SCSLabISU File: CDMSGD.py BSD 3-Clause "New" or "Revised" License | 4 votes |
|
def get_updates(self, params, constraints, loss):
pi=self.pi
n_agents=self.n_agents
agent_id=self.agent_id
parameters=globals()["parameters"]
info_shapes = [K.get_variable_shape(p) for p in params]
parameter_copy = [0 for i in range(n_agents)]
for nb in range(n_agents):
parameter_copy[nb]=parameters[nb]
if (nb==agent_id):
parameter_copy[nb]=params
grads = self.get_gradients(loss, params)
self.updates = []
lr = self.lr
if self.initial_decay > 0:
lr *= (1. / K.sqrt(1. + self.decay * self.iterations))
self.updates .append(K.update_add(self.iterations, 1))
# momentum
shapes = [K.get_variable_shape(p) for p in params]
moments = [K.zeros(shape) for shape in shapes]
dist_accumulator = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + moments
#for p, g, m, d, p_agents in zip(params, grads, moments, dist_accumulator, *parameter_copy):
for i in range(len(params)):
p=params[i]
g=grads[i]
m=moments[i]
d=dist_accumulator[i]
# Momentum term
#v = self.momentum * m - lr * g # velocity
v = self.momentum * m - lr * g # velocity
self.updates.append(K.update(m, v))
if self.nesterov:
for nb in range(n_agents):
d+=pi[nb][agent_id]*parameter_copy[nb][i]
new_p = d + self.momentum * v - lr * g
else:
# This is for Debug only
# if count>5:
# raise ValueError('parameters: ' + str(p1) + str(p2) + str(p3) + str(p4) + str(p5) )
for nb in range(n_agents):
d+=pi[nb][agent_id]*parameter_copy[nb][i]
new_p = d + v
# apply constraints
if p in constraints:
c = constraints[p]
new_p = c(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
Example 80
| Project: dynamic_memory_networks_with_keras Author: vchudinov File: attention_cells.py GNU General Public License v3.0 | 4 votes |
|
def __init__(self,
units,
activation='sigmoid',
batch_size=0,
recurrent_activation='hard_sigmoid',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
implementation=1,
return_sequences=False,
**kwargs):
"""Identical to keras.recurrent.GRUCell.
The difference comes from the computation in self.call
"""
super(SoftAttnGRU, self).__init__(**kwargs)
self.batch_size = batch_size
self.units = units
self.return_sequences = return_sequences
self.activation = activations.get(activation)
self.recurrent_activation = activations.get(recurrent_activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.recurrent_initializer = initializers.get(recurrent_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.recurrent_constraint = constraints.get(recurrent_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.dropout = min(1., max(0., dropout))
self.recurrent_dropout = min(1., max(0., recurrent_dropout))
self.implementation = implementation
self.state_size = self.units
self._dropout_mask = None
self._recurrent_dropout_mask = None
self._input_map = {}
super(SoftAttnGRU, self).__init__(**kwargs)
