# -*- coding: utf-8 -*-
"""
Created on Sun Aug 6 10:37:22 2017
@author: soujanyaporia
"""
import numpy as np
np.random.seed(1337) # for reproducibility
from keras import backend as K
from keras.layers.core import Lambda
from keras.engine.topology import Layer
from keras.models import Sequential
from keras.layers import Input,Dense,GRU,LSTM,Concatenate,Dropout,Activation,Add, Masking
from keras.layers.pooling import AveragePooling1D,MaxPooling1D
from keras.layers.core import Flatten
from keras.callbacks import EarlyStopping
from keras.layers.convolutional import Conv1D
from keras.models import Model
from keras.layers.wrappers import TimeDistributed, Bidirectional
from keras.layers.core import Reshape
from keras.backend import shape
from keras.utils import plot_model
from keras.layers.merge import Multiply,Concatenate
import pickle
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
from keras.optimizers import RMSprop,Adadelta,Adam
from keras.callbacks import Callback
import sys
def createOneHot(train_label, test_label):
maxlen = int(max(train_label.max(), test_label.max()))
train = np.zeros((train_label.shape[0], train_label.shape[1], maxlen+1))
test = np.zeros((test_label.shape[0], test_label.shape[1], maxlen+1))
for i in range(train_label.shape[0]):
for j in range(train_label.shape[1]):
train[i,j,train_label[i,j]]=1
for i in range(test_label.shape[0]):
for j in range(test_label.shape[1]):
test[i,j,test_label[i,j]]=1
return train, test
def calc_test_result(result, test_label, test_mask):
true_label=[]
predicted_label=[]
for i in range(result.shape[0]):
for j in range(result.shape[1]):
if test_mask[i,j]==1:
true_label.append(np.argmax(test_label[i,j] ))
predicted_label.append(np.argmax(result[i,j] ))
print("Confusion Matrix :")
print(confusion_matrix(true_label, predicted_label))
print("Classification Report :")
print(classification_report(true_label, predicted_label,digits=4))
print("Accuracy ", accuracy_score(true_label, predicted_label))
print("Macro Classification Report :")
print(precision_recall_fscore_support(true_label, predicted_label,average='macro'))
print("Weighted Classification Report :")
print(precision_recall_fscore_support(true_label, predicted_label,average='weighted'))
#print "Normal Classification Report :"
#print precision_recall_fscore_support(true_label, predicted_label)
def crop(a):
return a[:, 0:text_dim]
def crop1(a):
return a[:, text_dim:(text_dim+visual_dim)]
def crop2(a):
return a[:, (text_dim+visual_dim):dim]
def output_of_lambda(input_shape):
return (input_shape[0], text_dim)
def output_of_lambda1(input_shape):
return (input_shape[0], visual_dim)
def output_of_lambda2(input_shape):
return (input_shape[0], audio_dim)
# ################################################################################
# #Level 2
# #concatenate level 1 output to be sent to hfusion
# fused_tensor=Concatenate(axis=2)([context_1_2,context_1_3,context_2_3])
def load_bimodal_activations():
with open('./bimodal.pickle', 'rb') as handle:
activations = pickle.load(handle, encoding = 'latin1')
merged_train_data = activations['train_bimodal']
merged_test_data = activations['test_bimodal']
train_mask=activations['train_mask']
test_mask=activations['test_mask']
train_label=activations['train_label']
test_label=activations['test_label']
#Setting dummy utterances to be 0
for i in range(activations['train_bimodal'].shape[0]):
for j in range(activations['train_bimodal'].shape[1]):
if train_mask[i][j] == 0.0 :
merged_train_data[i,j,:]=0.0
for i in range(activations['test_bimodal'].shape[0]):
for j in range(activations['test_bimodal'].shape[1]):
if test_mask[i][j] == 0.0 :
merged_test_data[i,j,:]=0.0
audio_dim=500
visual_dim=500
text_dim=500
dim = audio_dim + visual_dim + text_dim
max_len = merged_train_data.shape[1] #max number of utterances per video
dim_proj=450
return merged_train_data, merged_test_data, train_label, test_label, train_mask, test_mask, max_len
def load_unimodal_activations():
with open('./unimodal.pickle', 'rb') as handle:
unimodal_activations = pickle.load(handle, encoding = 'latin1')
merged_train_data = np.concatenate((unimodal_activations['text_train'], unimodal_activations['audio_train'], unimodal_activations['video_train']), axis=2)
merged_test_data = np.concatenate((unimodal_activations['text_test'], unimodal_activations['audio_test'], unimodal_activations['video_test']), axis=2)
train_mask=unimodal_activations['train_mask']
test_mask=unimodal_activations['test_mask']
train_label=unimodal_activations['train_label']
test_label=unimodal_activations['test_label']
#Setting dummy utterances to be 0
for i in range(unimodal_activations['audio_train'].shape[0]):
for j in range(unimodal_activations['audio_train'].shape[1]):
if train_mask[i][j] == 0.0 :
merged_train_data[i,j,:]=0.0
for i in range(unimodal_activations['audio_test'].shape[0]):
for j in range(unimodal_activations['audio_test'].shape[1]):
if test_mask[i][j] == 0.0 :
merged_test_data[i,j,:]=0.0
global audio_dim, visual_dim, text_dim, dim, max_len, dim_proj
audio_dim=unimodal_activations['audio_train'].shape[2]
visual_dim=unimodal_activations['video_train'].shape[2]
text_dim=unimodal_activations['text_train'].shape[2]
dim = audio_dim + visual_dim + text_dim
max_len = merged_train_data.shape[1] #max number of utterances per video
dim_proj=450
return merged_train_data, merged_test_data, train_label, test_label, train_mask, test_mask
def Bimodal():
class hadamard2(Layer):
def __init__(self, prefix, **kwargs):
self.supports_masking = True
self.prefix = prefix
super(hadamard2, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.output_dim = dim_proj
self.kernel1 = self.add_weight(name=self.prefix+'kernel1',
shape=(self.output_dim,),
initializer='TruncatedNormal',
trainable=True)
self.kernel2 = self.add_weight(name=self.prefix+'kernel2',
shape=(self.output_dim,),
initializer='TruncatedNormal',
trainable=True)
self.bias = self.add_weight(name=self.prefix+'bias',
shape=(self.output_dim,),
initializer='zeros',
trainable=True)
super(hadamard2, self).build(input_shape) # Be sure to call this somewhere!
def call(self, x):
assert (K.int_shape(x[0])[1] <= dim_proj)
return K.tanh(x[0]*self.kernel1 + x[1]*self.kernel2 + self.bias)
def compute_output_shape(self, input_shape):
return (input_shape[0][0],self.output_dim)
merged_train_data, merged_test_data, train_label, test_label, train_mask, test_mask = load_unimodal_activations()
#################################################################################
#Level 1 sub-model : Hfusion
x=Input(shape=(audio_dim+visual_dim+text_dim,), name='bi_input')
masked = Masking(mask_value =0.0 , name='bi_mask')(x)
x1 = Lambda(crop,output_shape=output_of_lambda, name='bi_crop1')(masked)
x2 =Lambda(crop1,output_shape=output_of_lambda1, name='bi_crop2')(masked)
x3 =Lambda(crop2,output_shape=output_of_lambda2, name='bi_crop3')(masked)
dense1=Dense(dim_proj, activation='tanh', use_bias=False, trainable=True, name='bi_dense1')(x1)
dense2=Dense(dim_proj, activation='tanh', use_bias=False, trainable=True, name='bi_dense2')(x2)
dense3=Dense(dim_proj, activation='tanh', use_bias=False, trainable=True, name='bi_dense3')(x3)
fused_1_2=hadamard2('1')([dense1,dense2])
fused_1_3=hadamard2('2')([dense1,dense3])
fused_2_3=hadamard2('3')([dense2,dense3])
bimodal_1_2=Model(x,outputs=fused_1_2)
bimodal_1_3=Model(x,outputs=fused_1_3)
bimodal_2_3=Model(x,outputs=fused_2_3)
################################################################################
#Level 1 sub-model : biLSTM
input_data = Input(shape=(max_len, K.int_shape(fused_1_2)[1],), name='bi_lstm_input1')
lstm = GRU(600, activation='tanh', return_sequences = True, dropout=0.4, name='bi_lstm1')(input_data)
inter = Dropout(0.9, name='bi_dropout11')(lstm)
inter = TimeDistributed(Dense(500,activation='tanh', name='bi_tdis11'))(inter)
output1 = TimeDistributed(Dense(4,activation='softmax', name='bi_tdis12'))(inter)
lstm1 = Model(input_data, output1)
aux1 = Model(input_data, inter)
input_data = Input(shape=(max_len, K.int_shape(fused_1_2)[1],), name='bi_lstm_input2')
lstm = GRU(600, activation='tanh', return_sequences = True, dropout=0.4, name='bi_lstm2')(input_data)
inter = Dropout(0.9, name='bi_dropout21')(lstm)
inter = TimeDistributed(Dense(500,activation='tanh', name='bi_tdis21'))(inter)
output2 = TimeDistributed(Dense(4,activation='softmax', name='bi_tdis22'))(inter)
lstm2 = Model(input_data, output2)
aux2 = Model(input_data, inter)
input_data = Input(shape=(max_len, K.int_shape(fused_1_2)[1],), name='bi_lstm_input3')
lstm = GRU(600, activation='tanh', return_sequences = True, dropout=0.4, name='bi_lstm3')(input_data)
inter = Dropout(0.9, name='bi_dropout31')(lstm)
inter = TimeDistributed(Dense(500,activation='tanh', name='bi_tdis31'))(inter)
output3 = TimeDistributed(Dense(4,activation='softmax', name='bi_tdis32'))(inter)
lstm3 = Model(input_data, output3)
aux3 = Model(input_data, inter)
################################################################################
#Complete Level 1 : Lstm applied on hfusion
main_input=Input(shape=(max_len,audio_dim+visual_dim+text_dim,), name='bi_main_input')
video_uttr_1_2=TimeDistributed(bimodal_1_2, name='bi_tdis_b12')(main_input)
video_uttr_1_3=TimeDistributed(bimodal_1_3, name='bi_tdis_b13')(main_input)
video_uttr_2_3=TimeDistributed(bimodal_2_3, name='bi_tdis_b23')(main_input)
aux_1_2=aux1(video_uttr_1_2)
aux_1_3=aux2(video_uttr_1_3)
aux_2_3=aux3(video_uttr_2_3)
context_1_2=lstm1(video_uttr_1_2)
BM1 = Model(main_input, context_1_2)
Aux1 = Model(main_input, aux_1_2)
context_1_3=lstm2(video_uttr_1_3)
BM2 = Model(main_input, context_1_3)
Aux2 = Model(main_input, aux_1_3)
context_2_3=lstm3(video_uttr_2_3)
BM3 = Model(main_input, context_2_3)
Aux3 = Model(main_input, aux_2_3)
#Training and fitting of Bimodals
optimizer=Adam(lr=0.001)
BM1.compile(optimizer=optimizer, loss='categorical_crossentropy', sample_weight_mode='temporal')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
BM1.fit(merged_train_data, train_label,
epochs=200,
batch_size=20,
sample_weight=train_mask,
shuffle=True, callbacks=[early_stopping],
validation_split=0.1)
train_result1 = Aux1.predict(merged_train_data)
test_result1 = Aux1.predict(merged_test_data)
BM2.compile(optimizer=optimizer, loss='categorical_crossentropy', sample_weight_mode='temporal')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
BM2.fit(merged_train_data, train_label,
epochs=200,
batch_size=20,
sample_weight=train_mask,
shuffle=True, callbacks=[early_stopping],
validation_split=0.1)
train_result2 = Aux2.predict(merged_train_data)
test_result2 = Aux2.predict(merged_test_data)
BM3.compile(optimizer=optimizer, loss='categorical_crossentropy', sample_weight_mode='temporal')
early_stopping = EarlyStopping(monitor='val_loss', patience=10)
BM3.fit(merged_train_data, train_label,
epochs=200,
batch_size=20,
sample_weight=train_mask,
shuffle=True, callbacks=[early_stopping],
validation_split=0.1)
train_result3 = Aux3.predict(merged_train_data)
test_result3 = Aux3.predict(merged_test_data)
train_bimodal = np.concatenate((train_result1, train_result2, train_result3), axis=2)
test_bimodal = np.concatenate((test_result1, test_result2, test_result3), axis=2)
bimodal_activations={}
bimodal_activations['train_bimodal'] = train_bimodal
bimodal_activations['test_bimodal'] = test_bimodal
bimodal_activations['train_mask']=train_mask
bimodal_activations['test_mask']= test_mask
bimodal_activations['train_label']=train_label
bimodal_activations['test_label']=test_label
#merge all output
print("Saving bimodal activations")
with open('bimodal.pickle', 'wb') as handle:
pickle.dump(bimodal_activations, handle, protocol=pickle.HIGHEST_PROTOCOL)
def Trimodal():
merged_train_data, merged_test_data, train_label, test_label, train_mask, test_mask, \
maxlen = load_bimodal_activations()
#################################################################################
class TestCallback(Callback):
def __init__(self, test_data):
self.test_data = test_data
def on_epoch_end(self, epoch, logs={}):
x, y, z = self.test_data
result = model.predict(x)
calc_test_result(result, y, z)
class hadamard3(Layer):
def __init__(self, **kwargs):
self.supports_masking = True
super(hadamard3, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.output_dim = 500
self.kernel1 = self.add_weight(name='kernel1',
shape=(self.output_dim,),
initializer='glorot_uniform',
trainable=True)
self.kernel2 = self.add_weight(name='kernel2',
shape=(self.output_dim,),
initializer='glorot_uniform',
trainable=True)
self.kernel3 = self.add_weight(name='kernel3',
shape=(self.output_dim,),
initializer='glorot_uniform',
trainable=True)
self.bias = self.add_weight(name='bias',
shape=(self.output_dim,),
initializer='zeros',
trainable=True)
super(hadamard3, self).build(input_shape) # Be sure to call this somewhere!
def call(self, x, mask=None):
def crop(a):
return a[:, 0:500]
def crop1(a):
return a[:, 500:1000]
def crop2(a):
return a[:, 1000:1500]
def output_of_lambda(input_shape):
return (input_shape[0], 500)
def output_of_lambda1(input_shape):
return (input_shape[0], 500)
def output_of_lambda2(input_shape):
return (input_shape[0], 500)
x1=Lambda(crop,output_shape=output_of_lambda)(x)
x2=Lambda(crop1,output_shape=output_of_lambda1)(x)
x3=Lambda(crop2,output_shape=output_of_lambda2)(x)
ai = K.tanh(x1*self.kernel1 + x2*self.kernel2 + x3*self.kernel3 + self.bias)
return ai
def compute_output_shape(self, input_shape):
return (input_shape[0],self.output_dim)
def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
else:
return None
#################################################################################
#Level 2: Hfusion3 trimodal
x_tri=Input(shape=(1500,), name='tri_input')
fused_1_2_3=hadamard3()(x_tri)
hfusion_model=Model(x_tri,outputs=fused_1_2_3)
#################################################################################
#Complete Level 2 : Lstm applied on hfusion3
main_input= Input(shape=(maxlen,1500,), name='tri_main_input')
masked = Masking(mask_value =0.0, name='tri_masking')(main_input)
video_uttr=TimeDistributed(hfusion_model, name='tri_td_1')(masked)
lstm = GRU(500, activation='tanh', return_sequences = True, dropout=0.35, name='tri_lstm')(video_uttr)
lstm = Masking(mask_value=0.)(lstm)
inter = Dropout(0.86, name="tri_dropout1")(lstm)
concatenate_=Concatenate(name="tri_concat")([video_uttr,inter])
inter1_tri = TimeDistributed(Dense(450,activation='relu', name='tri_td_2'))(concatenate_)
inter1_tri = Masking(mask_value=0.)(inter1_tri)
context_fused_1_2_3 = Dropout(0.86, name="tri_dropout2")(inter1_tri)
context_fused_1_2_3 = Masking(mask_value=0.)(context_fused_1_2_3)
predictions = Dense(4, activation='softmax', name='tri_td_3')(context_fused_1_2_3)
#################################################################################
model=Model(main_input,outputs=predictions)
optimizer=Adam(lr=0.0001)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', sample_weight_mode='temporal')
early_stopping = EarlyStopping(monitor='val_loss', patience=5)
#print train_label.shape
print(train_label[0])
model.fit(merged_train_data, train_label,
epochs=100,
batch_size=20,
sample_weight=train_mask,
shuffle=True, callbacks=[TestCallback((merged_test_data,test_label,test_mask))],
validation_split=0.1)
result = model.predict(merged_test_data)
calc_test_result(result, test_label, test_mask)
if __name__ == '__main__':
#Bimodal()
Trimodal()
