import keras.backend as K
from keras import constraints
from keras import initializers
from keras import regularizers
from keras.engine.topology import Layer
class Attention(Layer):
def __init__(self,
W_regularizer=None,
b_regularizer=None,
W_constraint=None,
b_constraint=None,
bias=True, **kwargs):
"""
Keras Layer that implements an Content Attention mechanism.
Supports Masking.
"""
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
self.W_regularizer = regularizers.get(W_regularizer)
self.b_regularizer = regularizers.get(b_regularizer)
self.W_constraint = constraints.get(W_constraint)
self.b_constraint = constraints.get(b_constraint)
self.bias = bias
super(Attention, self).__init__(**kwargs)
def build(self, input_shape):
assert type(input_shape) == list
assert len(input_shape) == 2
self.steps = input_shape[0][1]
self.W = self.add_weight(shape=(input_shape[0][-1], input_shape[1][-1]),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.bias:
self.b = self.add_weight(shape=(1,),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
self.built = True
def compute_mask(self, input_tensor, mask=None):
return None
def call(self, input_tensor, mask=None):
x = input_tensor[0]
y = input_tensor[1]
mask = mask[0]
y = K.transpose(K.dot(self.W, K.transpose(y)))
y = K.expand_dims(y, axis=-2)
y = K.repeat_elements(y, self.steps, axis=1)
eij = K.sum(x * y, axis=-1)
if self.bias:
b = K.repeat_elements(self.b, self.steps, axis=0)
eij += b
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
return a
def get_output_shape_for(self, input_shape):
return (input_shape[0][0], input_shape[0][1])
def compute_output_shape(self, input_shape):
return input_shape[0][0], input_shape[0][1]
class WeightedSum(Layer):
def __init__(self, **kwargs):
self.supports_masking = True
super(WeightedSum, self).__init__(**kwargs)
def call(self, input_tensor, mask=None):
assert type(input_tensor) == list
assert type(mask) == list
x = input_tensor[0]
a = input_tensor[1]
a = K.expand_dims(a)
weighted_input = x * a
return K.sum(weighted_input, axis=1)
def get_output_shape_for(self, input_shape):
return (input_shape[0][0], input_shape[0][-1])
def compute_mask(self, x, mask=None):
return None
def compute_output_shape(self, input_shape):
return input_shape[0][0], input_shape[0][-1]
class WeightedAspectEmb(Layer):
def __init__(self, input_dim, output_dim,
init='uniform', input_length=None,
W_regularizer=None, activity_regularizer=None,
W_constraint=None,
weights=None, dropout=0., **kwargs):
self.input_dim = input_dim
self.output_dim = output_dim
self.init = initializers.get(init)
self.input_length = input_length
self.dropout = dropout
self.W_constraint = constraints.get(W_constraint)
self.W_regularizer = regularizers.get(W_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
if 0. < self.dropout < 1.:
self.uses_learning_phase = True
self.initial_weights = weights
kwargs['input_shape'] = (self.input_length,)
kwargs['input_dtype'] = K.floatx()
super(WeightedAspectEmb, self).__init__(**kwargs)
def build(self, input_shape):
self.W = self.add_weight(shape=(self.input_dim, self.output_dim),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
self.built = True
def compute_mask(self, x, mask=None):
return None
def get_output_shape_for(self, input_shape):
return (input_shape[0], self.output_dim)
def call(self, x, mask=None):
return K.dot(x, self.W)
def compute_output_shape(self, input_shape):
return input_shape[0], self.output_dim
def get_config(self):
config = super(WeightedAspectEmb, self).get_config()
config["input_dim"] = self.input_dim
config["output_dim"] = self.output_dim
return config
@classmethod
def from_config(cls, config):
input_dim, output_dim = config["input_dim"], config["output_dim"]
del config["input_dim"], config["output_dim"]
return cls(input_dim, output_dim, **config)
class Average(Layer):
def __init__(self, **kwargs):
self.supports_masking = True
super(Average, self).__init__(**kwargs)
def call(self, x, mask=None):
if mask is not None:
mask = K.cast(mask, K.floatx())
mask = K.expand_dims(mask)
x = x * mask
return K.sum(x, axis=-2) / K.sum(mask, axis=-2)
def get_output_shape_for(self, input_shape):
return input_shape[0:-2] + input_shape[-1:]
def compute_mask(self, x, mask=None):
return None
def compute_output_shape(self, input_shape):
return input_shape[0:-2] + input_shape[-1:]
class MaxMargin(Layer):
def __init__(self, **kwargs):
super(MaxMargin, self).__init__(**kwargs)
def call(self, input_tensor, mask=None):
z_s = input_tensor[0]
z_n = input_tensor[1]
r_s = input_tensor[2]
z_s = K.l2_normalize(z_s, axis=-1)
z_n = K.l2_normalize(z_n, axis=-1)
r_s = K.l2_normalize(r_s, axis=-1)
steps = z_n.shape[1]
pos = K.sum(z_s * r_s, axis=-1, keepdims=True)
pos = K.repeat_elements(pos, steps, axis=1)
r_s = K.expand_dims(r_s, axis=-2)
r_s = K.repeat_elements(r_s, steps, axis=1)
neg = K.sum(z_n * r_s, axis=-1)
loss = K.cast(K.sum(K.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx())
return loss
def compute_mask(self, input_tensor, mask=None):
return None
def get_output_shape_for(self, input_shape):
return (input_shape[0][0], 1)
def compute_output_shape(self, input_shape):
return input_shape[0][0], 1
