import keras.backend as K
from keras.engine.topology import Layer
from keras import initializers
from keras import regularizers
from keras import constraints
class Attention(Layer):
def __init__(self,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
"""
Keras Layer that implements an Content Attention mechanism.
Supports Masking.
"""
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
self.W_regularizer = regularizers.get(W_regularizer)
self.b_regularizer = regularizers.get(b_regularizer)
self.W_constraint = constraints.get(W_constraint)
self.b_constraint = constraints.get(b_constraint)
self.bias = bias
super(Attention, self).__init__(**kwargs)
def build(self, input_shape):
assert type(input_shape) == list
self.steps = input_shape[0][1]
self.W = self.add_weight((input_shape[0][-1], input_shape[1][-1]),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.bias:
self.b = self.add_weight((1,),
initializer='zeros',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
else:
self.b = None
self.built = True
def compute_mask(self, input_tensor, mask=None):
assert type(input_tensor) == list
assert type(mask) == list
return None
def call(self, input_tensor, mask=None):
x = input_tensor[0]
aspect = input_tensor[1]
mask = mask[0]
aspect = K.transpose(K.dot(self.W, K.transpose(aspect)))
aspect = K.expand_dims(aspect, axis=-2)
aspect = K.repeat_elements(aspect, self.steps, axis=1)
eij = K.sum(x*aspect, axis=-1)
if self.bias:
b = K.repeat_elements(self.b, self.steps, axis=0)
eij += b
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
return a
def compute_output_shape(self, input_shape):
return (input_shape[0][0], input_shape[0][1])
class WeightedSum(Layer):
def __init__(self, **kwargs):
self.supports_masking = True
super(WeightedSum, self).__init__(**kwargs)
def call(self, input_tensor, mask=None):
assert type(input_tensor) == list
assert type(mask) == list
x = input_tensor[0]
a = input_tensor[1]
a = K.expand_dims(a)
weighted_input = x * a
return K.sum(weighted_input, axis=1)
def compute_output_shape(self, input_shape):
return (input_shape[0][0], input_shape[0][-1])
def compute_mask(self, x, mask=None):
return None
class Average(Layer):
def __init__(self, mask_zero=True, **kwargs):
self.mask_zero = mask_zero
self.supports_masking = True
super(Average, self).__init__(**kwargs)
def call(self, x, mask=None):
if self.mask_zero:
mask = K.cast(mask, K.floatx())
mask = K.expand_dims(mask)
x = x * mask
return K.sum(x, axis=1) / (K.sum(mask, axis=1) + K.epsilon())
else:
return K.mean(x, axis=1)
def compute_output_shape(self, input_shape):
return (input_shape[0], input_shape[-1])
def compute_mask(self, x, mask):
return None
