import numpy as np from keras.applications.inception_v3 import InceptionV3 from keras.models import Sequential, load_model, Model from keras.layers import Input, Average, GlobalAveragePooling2D from keras.layers import TimeDistributed, GlobalAveragePooling1D from keras.layers.core import Dense, Dropout, Activation, Flatten from keras.layers.convolutional import Conv2D, MaxPooling2D from keras.optimizers import SGD, Adam from keras.layers.normalization import BatchNormalization class ResearchModels(): def __init__(self, nb_classes, n_snip, opt_flow_len, image_shape = (224, 224), saved_weights=None): """ `nb_classes` = the number of classes to predict `opt_flow_len` = the length of optical flow frames `image_shape` = shape of image frame `saved_model` = the path to a saved Keras model to load """ self.nb_classes = nb_classes self.n_snip = n_snip self.opt_flow_len = opt_flow_len self.saved_weights = saved_weights self.input_shape = (image_shape[0], image_shape[1], 3) self.input_shape_multi = (self.n_snip, image_shape[0], image_shape[1], 3) # Set the metrics. Only use top k if there's a need. metrics = ['accuracy'] if self.nb_classes >= 10: metrics.append('top_k_categorical_accuracy') self.model = self.cnn_spatial_multi() optimizer = Adam() self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics) # CNN model for the spatial stream with multiple inputs def cnn_spatial_multi(self): # shared cnn_spatial model cnn_spatial = self.cnn_spatial() cnn_spatial.load_weights(self.saved_weights) for layer in cnn_spatial.layers: layer.trainable = False # building inputs and output model = Sequential() model.add(TimeDistributed((cnn_spatial), input_shape=self.input_shape_multi)) model.add(GlobalAveragePooling1D()) return model # CNN model for the spatial stream def cnn_spatial(self, weights='imagenet'): # create the base pre-trained model base_model = InceptionV3(weights=weights, include_top=False) # add a global spatial average pooling layer x = base_model.output x = GlobalAveragePooling2D()(x) # let's add a fully-connected layer x = Dense(1024, activation='relu')(x) # and a logistic layer predictions = Dense(self.nb_classes, activation='softmax')(x) # this is the model we will train model = Model(inputs=base_model.input, outputs=predictions) return model