Python keras.losses.categorical_crossentropy() Examples

def nn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train.reshape(x_train.shape[0], -1) / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)
    # constant(value=1.)自定义常数，constant(value=1.)===one()
    # 创建模型:输入784个神经元，输出10个神经元
    model = Sequential([
        Dense(units=200, input_dim=784, bias_initializer=constant(value=1.), activation=tanh),
        Dense(units=100, bias_initializer=one(), activation=tanh),
        Dense(units=10, bias_initializer=one(), activation=softmax),
    ])

    opt = SGD(lr=0.2, clipnorm=1.)  # 优化器
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['acc', 'mae'])  # 编译
    model.fit(x_train, y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 

def main():
    input = Input(shape=(7, 2))
    output_1, state1_h, state1_c = LSTM(4, return_sequences=True,
                                        return_state=True)(input)
    output_2 = LSTM(4)(output_1, initial_state=[state1_h, state1_c])
    output_3 = LinkedAttention(250)([output_1, output_2])

    # state_h and state_c are only for the last timestamp.
    # output_1[-1] == state_h

    model = Model(inputs=[input], outputs=[output_1, output_3])
    model.compile(loss=categorical_crossentropy,
                  optimizer='adam',
                  metrics=['accuracy'])
    model.summary()
    # y = model.predict(np.ones((3, 7, 2)), batch_size=3)

    '''
    x = K.variable(value=np.array([[[1, 4]], [[-3, 2]]]))
    y = K.variable(value=np.array([[[1, 2, 3], [-1, 5, 2]],
                                  [[3, 4, 1], [1, 6, 4]]]))
    z = K.batch_dot(x, y)
    print(x.shape)
    print(K.eval(z))
    ''' 

def _calc_metric(self,y_true,y_pred):
        """
        Calculate evaluation metric.
        
        Supports: "roc-auc","norm-gini","mean_squared_error","mean_absolute_error",
                  "categorical_crossentropy","binary_crossentropy".
        """
        if self._val_loss=='roc-auc':
            metric = roc_auc_score(y_true, y_pred)
        elif self._val_loss=='norm-gini':
            metric = (2 * roc_auc_score(y_true, y_pred)) - 1
        elif self._val_loss=='mean_squared_error':
            metric = K.eval(mean_squared_error(K.variable(y_true), K.variable(y_pred)))
        elif self._val_loss=='mean_absolute_error':
            metric = K.eval(mean_absolute_error(K.variable(y_true), K.variable(y_pred)))
        elif self._val_loss=='categorical_crossentropy':
            metric = K.eval(categorical_crossentropy(K.variable(y_true), K.variable(y_pred)))
        elif self._val_loss=='binary_crossentropy':
            metric = K.eval(binary_crossentropy(K.variable(y_true), K.variable(y_pred)))
        else:
            raise ValueError('Invalid value for "custom_eval_stopping["name"], "roc-auc","norm-gini","mean_squared_error", \
                             "mean_absolute_error","categorical_crossentropy","binary_crossentropy" supported.')
        return metric 

def _get_metric(self,y_true,y_pred):
        """
        Calculate metric being logged.
        
        Supports: "roc-auc","norm-gini","mean_squared_error","mean_absolute_error",
                  "categorical_crossentropy","binary_crossentropy".
        """
        if self._metric=='roc-auc':
            metric = roc_auc_score(y_true, y_pred)
        elif self._metric=='norm-gini':
            metric = (2 * roc_auc_score(y_true, y_pred)) - 1
        elif self._metric=='mean_squared_error':
            metric = K.eval(mean_squared_error(K.variable(y_true), K.variable(y_pred)))
        elif self._metric=='mean_absolute_error':
            metric = K.eval(mean_absolute_error(K.variable(y_true), K.variable(y_pred)))
        elif self._metric=='categorical_crossentropy':
            metric = K.eval(categorical_crossentropy(K.variable(y_true), K.variable(y_pred)))
        elif self._metric=='binary_crossentropy':
            metric = K.eval(binary_crossentropy(K.variable(y_true), K.variable(y_pred)))
        else:
            raise ValueError('Invalid value for "custom_eval_stopping["name"], "roc-auc","norm-gini","mean_squared_error", \
                             "mean_absolute_error","categorical_crossentropy","binary_crossentropy" supported.')
        return metric 

def add_categorical_loss(model, number_of_classes):
    '''
    Adds categorical_crossentropy loss to an model.

    :params:
        model - Keras Model object
        number_of_classes - Integer, number of classes in a dataset (number of words in this case)

    :returns:
        model - Keras Model object with categorical_crossentropy loss added
    '''
    
    #Creates placeholder/Input layer for labels in one_hot_encoded form
    labels = Input(name='labels', shape=(number_of_classes,), dtype='float32')
    
    #Add categorical_crossentropy Loss to the input model
    loss = Lambda(categorical_loss, output_shape=(1,), name='categorical_crossentropy')([model.output, labels])
    
    #Create new model instance with all new placeholders/input layers and loss as the output
    model = Model(inputs=[model.input, labels], outputs=loss)
    
    return model 

def crf_loss(self, y_true, y_pred):
        """General CRF loss function depanding on the learning mode.
        
        # Arguments
            y_true: tensor with true targets.
            y_pred: tensor with predicted targets.
            
        # Returns
            If the CRF layer is being trained in the join mode, returns the negative
            log-likelihood. Otherwise returns the categorical crossentropy implemented
            by the underlying Keras backend.
            
        # About Codes
            This code is from Keras-Team/keras_contrib.losses.crf_losses, 
            change some details.
        """
        
        crf, idx = y_pred._keras_history[:2]
        if crf.learn_mode == "join":
            return self.crf_nll(y_true, y_pred)
        else:
            if crf.sparse_target:
                return sparse_categorical_crossentropy(y_true, y_pred)
            else:
                return categorical_crossentropy(y_true, y_pred) 

def crf_loss(self, y_true, y_pred):
        """General CRF loss function depanding on the learning mode.
        
        # Arguments
            y_true: tensor with true targets.
            y_pred: tensor with predicted targets.
            
        # Returns
            If the CRF layer is being trained in the join mode, returns the negative
            log-likelihood. Otherwise returns the categorical crossentropy implemented
            by the underlying Keras backend.
            
        # About Codes
            This code is from Keras-Team/keras_contrib.losses.crf_losses, 
            change some details.
        """
        
        crf, idx = y_pred._keras_history[:2]
        if crf.learn_mode == "join":
            return self.crf_nll(y_true, y_pred)
        else:
            if crf.sparse_target:
                return sparse_categorical_crossentropy(y_true, y_pred)
            else:
                return categorical_crossentropy(y_true, y_pred) 

def transfer_unit(model, training_data, training_labels,
                  testing_data, testing_labels, filename,
                  class_names=class_names):

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    # for layer in model.layers[:-7]:
    #     layer.trainable = False

    history = model.fit(training_data, training_labels,
                        batch_size=32, epochs=200, verbose=0,
                        validation_data=(testing_data, testing_labels))

    y_prob = model.predict(testing_data)
    values = get_metrics_and_plots(testing_labels, y_prob, history,
                                   filename, class_names)

    return values 

def transfer_unit(model, training_data, training_labels,
                  testing_data, testing_labels, filename,
                  class_names=class_names):

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    for layer in model.layers[:-7]:
        layer.trainable = False

    history = model.fit(training_data, training_labels,
                        batch_size=32, epochs=50, verbose=0,
                        validation_data=(testing_data, testing_labels))
    y_prob = model.predict(testing_data)
    values = get_metrics_and_plots(testing_labels, y_prob, history,
                                   filename, class_names)

    return values 

def transfer_unit(model, training_data, training_labels,
                  testing_data, testing_labels, filename,
                  class_names=class_names):

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    for layer in model.layers[:-7]:
        layer.trainable = False

    history = model.fit(training_data, training_labels,
                        batch_size=32, epochs=500, verbose=0,
                        validation_data=(testing_data, testing_labels))
    y_prob = model.predict(testing_data)
    values = get_metrics_and_plots(testing_labels, y_prob, history,
                                   filename, class_names)

    return values 

def load_and_build_models(student_model_file=STUDENT_MODEL_FILE,
                          teacher_model_file=TEACHER_MODEL_FILE,
                          cutoff_layer=CUTOFF_LAYER):

    # load the student model
    print('loading student model')
    student_model = load_model(student_model_file)

    for idx, layer in enumerate(student_model.layers):
        layer.trainable = True

    print('loading teacher model')
    teacher_model = load_model(teacher_model_file)

    # load the bottleneck model
    print('building bottleneck model')
    bottleneck_model = Model(teacher_model.input,
                             teacher_model.layers[cutoff_layer - 1].output)
    bottleneck_model.compile(loss='categorical_crossentropy',
                             optimizer='adam',
                             metrics=['accuracy'])

    student_model = recompile_student_model(student_model)

    return bottleneck_model, student_model 

def crf_loss(y_true, y_pred):
    """General CRF loss function depending on the learning mode.

    # Arguments
        y_true: tensor with true targets.
        y_pred: tensor with predicted targets.

    # Returns
        If the CRF layer is being trained in the join mode, returns the negative
        log-likelihood. Otherwise returns the categorical crossentropy implemented
        by the underlying Keras backend.

    # About GitHub
        If you open an issue or a pull request about CRF, please
        add `cc @lzfelix` to notify Luiz Felix.
    """
    crf, idx = y_pred._keras_history[:2]
    if crf.learn_mode == 'join':
        return crf_nll(y_true, y_pred)
    else:
        if crf.sparse_target:
            return sparse_categorical_crossentropy(y_true, y_pred)
        else:
            return categorical_crossentropy(y_true, y_pred) 

def create_model(self):
        self.model = Sequential()
        self.model.add(Conv2D(32, (3, 3), padding='same',
                              input_shape=self.shape,
                              activation='relu'))
        self.model.add(Conv2D(32, (3, 3), activation='relu'))
        self.model.add(MaxPooling2D(pool_size=(2, 2)))
        self.model.add(Dropout(0.25))

        self.model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
        self.model.add(Conv2D(64, (3, 3), activation='relu'))
        self.model.add(MaxPooling2D(pool_size=(2, 2)))
        self.model.add(Dropout(self.dropout_probability))

        self.model.add(Flatten())
        self.model.add(Dense(self.hidden_neurons, activation='relu'))
        self.model.add(Dropout(self.dropout_probability))
        self.model.add(Dense(self.num_classes, activation='softmax'))

        self.model.compile(loss=categorical_crossentropy,
                           optimizer=rmsprop(lr=self.learning_rate, decay=1e-6),
                           metrics=['accuracy']) 

def crf_loss(y_true, y_pred):
    """General CRF loss function depending on the learning mode.
    # Arguments
        y_true: tensor with true targets.
        y_pred: tensor with predicted targets.
    # Returns
        If the CRF layer is being trained in the join mode, returns the negative
        log-likelihood. Otherwise returns the categorical crossentropy implemented
        by the underlying Keras backend.
    # About GitHub
        If you open an issue or a pull request about CRF, please
        add `cc @lzfelix` to notify Luiz Felix.
    """
    crf, idx = y_pred._keras_history[:2]
    if crf.learn_mode == 'join':
        return crf_nll(y_true, y_pred)
    else:
        if crf.sparse_target:
            return sparse_categorical_crossentropy(y_true, y_pred)
        else:
            return categorical_crossentropy(y_true, y_pred)

# crf_marginal_accuracy, crf_viterbi_accuracy 

def iris():

    from keras.optimizers import Adam, Nadam
    from keras.losses import logcosh, categorical_crossentropy
    from keras.activations import relu, elu, softmax

    # here use a standard 2d dictionary for inputting the param boundaries
    p = {'lr': (0.5, 5, 10),
         'first_neuron': [4, 8, 16, 32, 64],
         'hidden_layers': [0, 1, 2, 3, 4],
         'batch_size': (2, 30, 10),
         'epochs': [2],
         'dropout': (0, 0.5, 5),
         'weight_regulizer': [None],
         'emb_output_dims':  [None],
         'shapes': ['brick', 'triangle', 0.2],
         'optimizer': [Adam, Nadam],
         'losses': [logcosh, categorical_crossentropy],
         'activation': [relu, elu],
         'last_activation': [softmax]}

    return p 

def knowledge_distillation_loss(y_true, y_pred, temperature):    
    
    # split in 
    #    true targets
    #    logits from xception
    y_true, logits = y_true[:, :256], y_true[:, 256:]
    
    # convert logits to soft targets
    y_soft = K.softmax(logits/temperature)
    
    # split in 
    #    usual output probabilities
    #    probabilities made softer with temperature
    y_pred, y_pred_soft = y_pred[:, :256], y_pred[:, 256:]    
    
    return K.in_train_phase(logloss(y_soft, y_pred_soft), logloss(y_true, y_pred))
    

# # For testing use usual output probabilities (without temperature) 

def knowledge_distillation_loss(y_true, y_pred, lambda_const, temperature):    
    
    # split in 
    #    onehot hard true targets
    #    logits from xception
    y_true, logits = y_true[:, :256], y_true[:, 256:]
    
    # convert logits to soft targets
    y_soft = K.softmax(logits/temperature)
    
    # split in 
    #    usual output probabilities
    #    probabilities made softer with temperature
    y_pred, y_pred_soft = y_pred[:, :256], y_pred[:, 256:]    
    
    return K.in_train_phase(logloss(y_true, y_pred), logloss(y_soft, y_pred_soft))
    # return lambda_const*logloss(y_true, y_pred) + logloss(y_soft, y_pred_soft)
    # return logloss(y_soft, y_pred_soft)


# # For testing use usual output probabilities (without temperature)

# In[13]: 

def create_model(input_shape: tuple, nb_classes: int, init_with_imagenet: bool = False, learning_rate: float = 0.01):
    weights = None
    if init_with_imagenet:
        weights = "imagenet"

    model = VGG16(input_shape=input_shape,
                  classes=nb_classes,
                  weights=weights,
                  include_top=False)
    # "Shallow" VGG for Cifar10
    x = model.get_layer('block3_pool').output
    x = layers.Flatten(name='Flatten')(x)
    x = layers.Dense(512, activation='relu')(x)
    x = layers.Dense(nb_classes)(x)
    x = layers.Softmax()(x)
    model = models.Model(model.input, x)

    loss = losses.categorical_crossentropy
    optimizer = optimizers.SGD(lr=learning_rate, decay=0.99)

    model.compile(optimizer, loss, metrics=["accuracy"])
    return model 

def classify(classifier_weights, gen_path, dis_path):
    """
    Function: classify  
    This function classifies output images from emoji GAN by using trained CNN-based classifier.  

    Input: classifier_weights
    Output: loss and accuracy  
    """
    img_shape = (64, 64, 3)
    num_classes = 82
    model = build_classifier(img_shape, num_classes)
    model.compile(loss=categorical_crossentropy,
                optimizer=Adadelta(),
                metrics=['accuracy'])
    model.load_weights(classifier_weights)
    x_test, y_test = emoji_gan_generator(gen_path, dis_path)
    result = model.evaluate(x=x_test, y=y_test, batch_size=26)
    print('Accuracy: {0}'.format(result[1]))
    return result[0], result[1] 

def main():
    init_config()
    sample_rate, seed = 0.8, 2019
    data_dir, file_ptn = "./dataset/data", "imdb|wiki"
    dataframes = reload_data(data_dir, file_ptn)
    trainset, testset = train_test_split(dataframes, train_size=sample_rate, test_size=1-sample_rate, random_state=seed)
    train_gen = preprocessing(trainset)
    validation_gen = preprocessing(testset)
    model_name = "./model/mobilev2.h5"
    pmodel = restore_model(model_name)
    pmodel.compile(optimizer="Adam",
        loss=["categorical_crossentropy", smooth_l1_fl_loss, "categorical_crossentropy"],
        metrics={"gender": "acc", "age": metrix_age_diff, "age_identy": kmse},
        loss_weights=[50, 0.5, 0]
    )
    batch_size = 32
    callbacks = [ModelCheckpoint(model_name, monitor='val_age_metrix_age_diff', verbose=1, save_best_only=False, mode='max')]
    history = pmodel.fit_generator(train_gen, steps_per_epoch=len(trainset) / batch_size, epochs=200, callbacks=callbacks, validation_data=validation_gen, validation_steps=len(testset) / batch_size * 3) 

def evaluate_model(model, X, y, scaler):

    y_pred = np.zeros_like(y)
    for ii in range(len(X)):
        X_sample = np.expand_dims(scaler.transform(X[ii]), axis=0)
        y_pred[ii] = model.predict_on_batch(X_sample)

    print(y.shape, y_pred.shape)
    y = K.variable(y)
    y_pred = K.variable(y_pred)

    loss = K.eval(categorical_crossentropy(y, y_pred))

    return np.mean(loss) 

def cnn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train.reshape(-1, 28, 28, 1) / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)

    model = Sequential([
        # input_shape:输入平面，就在第一个位置设置
        # filters：卷积核、滤波器
        # kernel_size：卷积核大小
        # strides：步长
        # padding有两种方式：same/valid
        # activation：激活函数
        Convolution2D(input_shape=(28, 28, 1), filters=32, kernel_size=5, strides=1, padding='same', activation=relu),
        MaxPool2D(pool_size=2, strides=2, padding='same'),
        Convolution2D(filters=64, kernel_size=5, padding='same', activation=relu),
        MaxPool2D(pool_size=2, trainable=2, padding='same'),
        Flatten(),  # 扁平化
        Dense(units=1024, activation=relu),
        Dropout(0.5),
        Dense(units=10, activation=softmax),
    ])
    opt = Adam(lr=1e-4)
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
    model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 

def rnn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)

    model = Sequential([
        SimpleRNN(units=50, input_shape=(28, 28)),
        Dense(units=10, activation=softmax),
    ])
    opt = RMSprop(lr=1e-4)
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
    model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 

def compile_default(self):
        self.compile("adam",
                     loss=categorical_crossentropy,
                     metrics=[categorical_accuracy]) 

def compile_default(self):
        self.compile("adam",
                     loss=categorical_crossentropy,
                     metrics=[categorical_accuracy]) 

def compile_default(self):
        self.compile(optimizer="adam", 
                     loss=lambda y_true, y_pred: categorical_crossentropy(y_true, y_pred),# + 10000.0, #self.l2(y_true, y_pred), 
                     metrics=[categorical_accuracy]) 

def compile_default(self):
        self.compile(optimizer="adam", 
                     loss=categorical_crossentropy,# + 10000.0, #self.l2(y_true, y_pred), 
                     metrics=[categorical_accuracy]) 

def test_similarity_layer1(self):
        np.random.seed(181)

        input1 = Input(shape=(2, 3), name="input1")
        input2 = Input(shape=(4, 3), name="input2")

        output = Similarity()([input1, input2])

        model = Model(inputs=[input1, input2], outputs=[output])
        model.compile(loss=categorical_crossentropy,
                      optimizer='adam',
                      metrics=['accuracy'])

        x = np.array([
                [[1, 2, 3], [4, 5, 6]],
                [[-0.5, -1, -1.5], [-2, -2.5, -3]]
        ])
        y = np.array([
                [[7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]],
                [[-13, -14, -15], [-7, -8, -9], [-10, -11, 0], [-16, -17, -18]]
        ])

        rez = model.predict({
            'input1': x,
            'input2': y
        }, batch_size=2)
        assert(rez.shape == (2, 2, 4))

        WS = np.array([0.4666319, 0.7074857, -0.14512873, 0.00937462,
                      -0.225935, -0.02118444, 0.31465364, -0.75302243,
                       0.5018892])

        correct = self._do_brute_force(x, y, WS)
        self.assertTrue(np.allclose(rez, correct)) 

def test_similarity_layer2(self):
        np.random.seed(181)

        input1 = Input(shape=(20, 3), name="input1")
        input2 = Input(shape=(55, 3), name="input2")

        output = Similarity()([input1, input2])

        model = Model(inputs=[input1, input2], outputs=[output])
        model.compile(loss=categorical_crossentropy,
                      optimizer='adam',
                      metrics=['accuracy'])

        x = np.random.rand(2, 20, 3)
        y = np.random.rand(2, 55, 3)

        rez = model.predict({
            'input1': x,
            'input2': y
        }, batch_size=2)
        assert(rez.shape == (2, 20, 55))

        WS = np.array([0.4666319, 0.7074857, -0.14512873, 0.00937462,
                      -0.225935, -0.02118444, 0.31465364, -0.75302243,
                       0.5018892])

        correct = self._do_brute_force(x, y, WS)
        self.assertTrue(np.allclose(rez, correct, rtol=1e-03, atol=1e-04)) 

def test_similarity_layer3(self):
        np.random.seed(181)

        input1 = Input(shape=(20, 3), name="input1")
        input2 = Input(shape=(55, 3), name="input2")

        output = Similarity()([input1, input2])

        model = Model(inputs=[input1, input2], outputs=[output])
        model.compile(loss=categorical_crossentropy,
                      optimizer='adam',
                      metrics=['accuracy'])

        x = np.random.rand(2, 20, 3)
        y = np.random.rand(2, 55, 3)

        rez = model.predict({
            'input1': x,
            'input2': y
        }, batch_size=1)
        assert(rez.shape == (2, 20, 55))

        WS = np.array([0.4666319, 0.7074857, -0.14512873, 0.00937462,
                      -0.225935, -0.02118444, 0.31465364, -0.75302243,
                       0.5018892])

        correct = self._do_brute_force(x, y, WS)
        self.assertTrue(np.allclose(rez, correct, rtol=1e-03, atol=1e-04)) 

def categorical_loss(args):
    '''
    Creates Categorical crossentropy loss for a classificaiton based model.

    :params:
        args - List of params: predictions, labels

    :returns:
        calculated categorical_crossentropy loss based on args.
    '''

    predictions, labels = args
    return categorical_crossentropy(labels, predictions) 

def loss(y_true, y_pred):
    weights = K.cast(K.abs(K.argmax(y_true, axis=1) - K.argmax(y_pred, axis=1))/(K.int_shape(y_pred)[1] - 1), dtype='float32')
    return (1.0 + weights) * losses.categorical_crossentropy(y_true, y_pred) 

def test_functional_model_saving():
    img_rows, img_cols = 32, 32
    img_channels = 3

    # Parameters for the DenseNet model builder
    img_dim = (img_channels, img_rows, img_cols) if K.image_data_format() == 'channels_first' else (
        img_rows, img_cols, img_channels)
    depth = 40
    nb_dense_block = 3
    growth_rate = 3  # number of z2 maps equals growth_rate * group_size, so keep this small.
    nb_filter = 16
    dropout_rate = 0.0  # 0.0 for data augmentation
    conv_group = 'D4'  # C4 includes 90 degree rotations, D4 additionally includes reflections in x and y axis.
    use_gcnn = True

    # Create the model (without loading weights)
    model = GDenseNet(mc_dropout=False, padding='same', nb_dense_block=nb_dense_block, growth_rate=growth_rate,
                      nb_filter=nb_filter, dropout_rate=dropout_rate, weights=None, input_shape=img_dim, depth=depth,
                      use_gcnn=use_gcnn, conv_group=conv_group)
    model.compile(loss=losses.categorical_crossentropy,
                  optimizer=optimizers.Adam(),
                  metrics=[metrics.categorical_accuracy])
    x = np.random.random((1, 32, 32, 3))
    y = np.random.randint(0, 10, 1)
    y = np_utils.to_categorical(y, 10)
    model.train_on_batch(x, y)

    out = model.predict(x)
    _, fname = tempfile.mkstemp('.h5')
    save_model(model, fname)

    model = load_model(fname)
    os.remove(fname)

    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05) 

def test_weighted_masked_objective():
    a = Input(shape=(3,), name='input_a')

    # weighted_masked_objective
    def mask_dummy(y_true=None, y_pred=None, weight=None):
        return K.placeholder(y_true.shape)

    weighted_function = training_utils.weighted_masked_objective(
        losses.categorical_crossentropy)
    weighted_function(a, a, None) 

def test_check_not_failing():
    a = np.random.random((2, 1, 3))
    training_utils.check_loss_and_target_compatibility(
        [a], [losses.categorical_crossentropy], [a.shape])
    training_utils.check_loss_and_target_compatibility(
        [a], [losses.categorical_crossentropy], [(2, None, 3)]) 

def test_check_last_is_one():
    a = np.random.random((2, 3, 1))
    with pytest.raises(ValueError) as exc:
        training_utils.check_loss_and_target_compatibility(
            [a], [losses.categorical_crossentropy], [a.shape])

    assert 'You are passing a target array' in str(exc) 

def test_check_bad_shape():
    a = np.random.random((2, 3, 5))
    with pytest.raises(ValueError) as exc:
        training_utils.check_loss_and_target_compatibility(
            [a], [losses.categorical_crossentropy], [(2, 3, 6)])

    assert 'targets to have the same shape' in str(exc) 

def build_model(self, input_shape, n_classes):
        # Load the VGG model
        xception_conv = Xception(weights='imagenet', include_top=False, input_shape=input_shape)

        # Freeze the layers except the last 4 layers
        for layer in xception_conv.layers:
            layer.trainable = False

        # Create the model
        model = models.Sequential()

        # Add the vgg convolutional base model
        model.add(xception_conv)
        # Add new layers
        model.add(Flatten())
        model.add(Dense(1024, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(n_classes, activation='softmax', name='predictions'))

        # define the model with input layer and output layer
        model.summary()
        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)
        model.compile(loss=categorical_crossentropy, optimizer=Adam(lr=0.01), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)

        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        cnn_1 = self.cnn(input_shape)
        input_1 = cnn_1.input
        output_1 = cnn_1.output
        cnn_2 = self.cnn(input_shape)
        input_2 = cnn_2.input
        output_2 = cnn_2.output
        cnn_3 = self.cnn(input_shape)
        input_3 = cnn_3.input
        output_3 = cnn_3.output
        cnn_4 = self.cnn(input_shape)
        input_4 = cnn_4.input
        output_4 = cnn_4.output

        concat_layer = Maximum()([output_1, output_2, output_3, output_4])
        concat_layer = Dropout(0.5)(concat_layer)
        dense_layer1 = Dense(units=1024, activation='relu')(concat_layer)
        dense_layer1 = Dropout(0.5)(dense_layer1)
        output_layer = Dense(n_classes, activation='softmax', name='predictions')(dense_layer1)

        model = Model(inputs=[input_1, input_2, input_3, input_4], outputs=[output_layer])
        model.summary()
        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)
        model.compile(loss=categorical_crossentropy, optimizer=Adadelta(lr=0.01), metrics=['accuracy'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        ## input layer
        input_layer = Input(input_shape)

        ## convolutional layers
        conv_layer1 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu')(input_layer)
        pooling_layer1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))(conv_layer1)

        conv_layer2 = Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(pooling_layer1)

        conv_layer3 = Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(conv_layer2)
        pooling_layer2 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))(conv_layer3)
        dropout_layer =Dropout(0.5)(pooling_layer2)

        dense_layer = Dense(units=2048, activation='relu')(dropout_layer)
        output_layer = Dense(units=n_classes, activation='softmax')(dense_layer)

        ## define the model with input layer and output layer
        model = Model(inputs=input_layer, outputs=output_layer)
        model.summary()

        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)

        model.compile(loss=categorical_crossentropy, optimizer=Adam(), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def crossentropy(y_true, y_pred):

    from keras.losses import categorical_crossentropy
    entropy = categorical_crossentropy(y_true, y_pred)

    return entropy 

def compute_egls(self, unlabeled, n_classes):

        # create a function for computing the gradient length:
        self.input_placeholder = K.placeholder(self.model.get_input_shape_at(0))
        self.output_placeholder = K.placeholder(self.model.get_output_shape_at(0))
        predict = self.model.call(self.input_placeholder)
        loss = K.mean(categorical_crossentropy(self.output_placeholder, predict))
        weights = [tensor for tensor in self.model.trainable_weights]
        gradient = self.model.optimizer.get_gradients(loss, weights)
        gradient_flat = [K.flatten(x) for x in gradient]
        gradient_flat = K.concatenate(gradient_flat)
        gradient_length = K.sum(K.square(gradient_flat))
        self.get_gradient_length = K.function([K.learning_phase(), self.input_placeholder, self.output_placeholder], [gradient_length])

        # calculate the expected gradient length of the unlabeled set (iteratively, to avoid memory issues):
        unlabeled_predictions = self.model.predict(unlabeled)
        egls = np.zeros(unlabeled.shape[0])
        for i in range(n_classes):
            calculated_so_far = 0
            while calculated_so_far < unlabeled_predictions.shape[0]:
                if calculated_so_far + 100 >= unlabeled_predictions.shape[0]:
                    next = unlabeled_predictions.shape[0] - calculated_so_far
                else:
                    next = 100

                labels = np.zeros((next, n_classes))
                labels[:,i] = 1
                grads = self.get_gradient_length([0, unlabeled[calculated_so_far:calculated_so_far+next, :], labels])[0]
                grads *= unlabeled_predictions[calculated_so_far:calculated_so_far+next, i]
                egls[calculated_so_far:calculated_so_far+next] += grads

                calculated_so_far += next

        return egls 

def cce_loss_graph(gt, pr):
    return K.mean(categorical_crossentropy(gt, pr)) 

def crf_loss(y_true, y_pred):
    """General CRF loss function, depending on the learning mode."""
    crf, idx = y_pred._keras_history[:2]
    if crf.learn_mode == 'join':
        return crf_nll(y_true, y_pred)
    else:
        if crf.sparse_target:
            return sparse_categorical_crossentropy(y_true, y_pred)
        else:
            return categorical_crossentropy(y_true, y_pred) 

def loss(y_true, y_pred):
    entropy = -K.mean(K.sum(y_pred*K.log(y_pred), 1))
    beta = 0.1
    return logloss(y_true, y_pred) - beta*entropy 

def categorical_crossentropy(y_true, y_pred):
    y_true = y_true[:, :256]
    y_pred = y_pred[:, :256]
    return logloss(y_true, y_pred)


# logloss with only soft probabilities and targets 

def soft_logloss(y_true, y_pred):     
    logits = y_true[:, 256:]
    y_soft = K.softmax(logits/temperature)
    y_pred_soft = y_pred[:, 256:]    
    return logloss(y_soft, y_pred_soft) 

def soft_logloss(y_true, y_pred):     
    logits = y_true[:, 256:]
    y_soft = K.softmax(logits/temperature)
    y_pred_soft = y_pred[:, 256:]    
    return logloss(y_soft, y_pred_soft)


# # Train

# In[17]:


# lambda_const = 0.2 

def compile(self):
        self.model.compile(
            loss=losses.categorical_crossentropy,
            optimizer=LangModelSGD(self.setting),
            metrics=["accuracy", self.perplexity]
            ) 

def perplexity(cls, y_true, y_pred):
        cross_entropy = K.mean(K.categorical_crossentropy(y_pred, y_true), axis=-1)
        perplexity = K.exp(cross_entropy)
        return perplexity 

def dice_cross_loss(y_true, y_pred):
    return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*dice_coef_loss(y_true, y_pred) 

def dice_cross_loss(y_true, y_pred):
    return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*dice_coef_loss(y_true, y_pred) 

def dice_cross_loss(y_true, y_pred):
    return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*cross_dice_coef_loss(y_true, y_pred)

    # return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*cross_dice_coef_loss(y_true, y_pred) 

def dice_cross_loss(y_true, y_pred):
    return 0.9*categorical_crossentropy(y_true,y_pred) + 0.1*dice_coef_loss(y_true, y_pred) 

def dice_cross_loss(y_true, y_pred):
    return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*cross_dice_coef_loss(y_true, y_pred)

    # return 0.8*categorical_crossentropy(y_true,y_pred) + 0.2*cross_dice_coef_loss(y_true, y_pred) 

def prepare_conv2d(x_train, y_train, x_test, y_test):
    num_tasks = x_train.shape[1]
    batch_size = 10
    epochs = 10
    input_shape = (num_tasks, 3, 1)
    num_classes = num_tasks
    model = Sequential()
    model.add(Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=input_shape))
    model.add(Conv2D(64, (1, 1), activation='relu'))
    model.add(MaxPooling2D(pool_size=(1, 1)))
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_tasks * num_classes, activation='softmax'))

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adadelta(),
                  metrics=['accuracy'])

    flatten_categorized_y_train = get_flat_categorized_y(y_train, num_classes)
    flatten_categorized_y_test = get_flat_categorized_y(y_test, num_classes)

    history = model.fit(x_train, flatten_categorized_y_train,
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=1,
                        validation_data=(x_test, flatten_categorized_y_test))
    visualize_training(history)
    # score = model.evaluate(x_test, flatten_categorized_y_train, verbose=0)
    # print('Test loss:', score[0])
    # print('Test accuracy:', score[1])
    classification = model.predict(x_test, batch_size=batch_size)
    classification = [c.reshape(num_tasks, num_classes) for c in classification]
    return classification, model 

def __init__(self, bert_config, pretrain_model_path, batch_size, seq_length, optimizer, num_classes, metrics=None,
                 use_token_type=True, mask=True, max_predictions_per_seq=20, multi_gpu=None, loss=None):
        if not isinstance(bert_config, BertConfig):
            raise ValueError("`bert_config` must be a instance of `BertConfig`")
        if multi_gpu:
            if not tf.test.is_gpu_available:
                raise ValueError("GPU is not available.")

        self.config = bert_config
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.use_token_type = use_token_type
        self.max_predictions_per_seq = max_predictions_per_seq
        self.mask = mask
        self.num_classes = num_classes
        self.loss = loss or losses.categorical_crossentropy

        if multi_gpu:
            with tf.device('/cpu:0'):
                model = self._build_model(pretrain_model_path)
                model.compile(optimizer=optimizer, loss=self.loss, metrics=metrics)
            parallel_model = multi_gpu_model(model=model, gpus=multi_gpu)
            parallel_model.compile(optimizer=optimizer, loss=self.loss, metrics=metrics)
        else:
            model = self._build_model(pretrain_model_path)
            model.compile(optimizer=optimizer, loss=self.loss, metrics=metrics)

        self.estimator = model
        if multi_gpu:
            self.estimator = parallel_model 

def train():
    """
    Function: train  
    This function trains CNN-based model.  

    Input: None  
    Output: None (model weight file will be saved.)   
    """
    img_shape = (64, 64, 3)
    num_classes = 82
    epochs = 100
    model = build_classifier(img_shape, num_classes)
    model.summary()
    
    (x_train, y_train), (x_test, y_test) = make_label()

    model.compile(loss=categorical_crossentropy,
              optimizer=Adadelta(),
              metrics=['accuracy'])

    callbacks = []
    callbacks.append(CSVLogger('./saved_model/history_classifier.csv'))
    callbacks.append(EarlyStopping(patience=2, verbose=1))

    history = model.fit(x=x_train, y=y_train, 
        batch_size=26, epochs=epochs, verbose=1,
        validation_data=(x_test, y_test), callbacks=callbacks)
    
    model.save_weights('./saved_model/classifier_weights.h5') 

def categorical_crossentropy_with_logit(y_true, y_pred):
	return K.categorical_crossentropy(y_true,y_pred,from_logits=True) 

def smooth_l1_ce_loss(y_true, y_pred):
    # we use smoth l1 for adding constance relation of age
    ture_arg_max = K.cast(K.argmax(y_true, axis = -1), "float32")
    pred_arg_max = K.cast(K.argmax(y_pred, axis = -1), "float32")
    diff = K.abs(ture_arg_max - pred_arg_max)
    ls = K.switch(diff < 3, (0.3 * diff * diff), (diff - 0.3)) * categorical_crossentropy(y_true, y_pred)
    return K.mean(ls) 

def smooth_l1_fl_loss(y_true, y_pred):
    # we use smoth l1 for adding constance relation of age focal loss version
    ture_arg_max = K.cast(K.argmax(y_true, axis = -1), "float32")
    pred_arg_max = K.cast(K.argmax(y_pred, axis = -1), "float32")
    factor_alpha = (1 - K.sum(y_true * y_pred, axis=-1))
    factor_alpha = factor_alpha * factor_alpha
    factor_offset = K.abs(ture_arg_max - 25) // 5 + 1
    diff = K.abs(ture_arg_max - pred_arg_max)
    ls = K.switch(diff < 3, (0.3 * diff * diff), (diff - 0.3)) * factor_offset * factor_alpha * categorical_crossentropy(y_true, y_pred)
    return K.mean(ls) 

def _build_net(self) -> keras.models.Model:
        input_tensor = keras.layers.Input(shape=self.inputshape())
        hidden_tensor = keras.layers.Dense(800, activation=relu)(input_tensor)

        pitch_tensor = keras.layers.Dense(127, activation=softmax)(hidden_tensor)
        tsbq_tensor = keras.layers.Dense(self.maxtsbq + 1, activation=softmax)(hidden_tensor)
        dq_tensor = keras.layers.Dense(self.maxdq + 1, activation=softmax)(hidden_tensor)

        model = keras.models.Model(inputs=input_tensor, outputs=[pitch_tensor, tsbq_tensor, dq_tensor])
        model.compile(optimizer=adagrad(), loss=categorical_crossentropy)

        self.epochs = 20
        self.outfuns = sampler(.3), weighted_nlargest(2), weighted_nlargest(2)
        return model 

def simple_LSTM(num_words, embeddings_matrix, ce_loader,
                scope, embedding_dim=64):
    # (batch, input_len) => (batch, input_len, embedding_dim)
    q_input = Input(shape=(QUESTION_LEN,), name="q_input")
    a_input = Input(shape=(ANSWER_LEN,), name="a_input")
    c_input = Input(shape=(CONTEXT_LEN,), name="c_input")

    q_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=QUESTION_LEN,
                      name="embedding_q_" + scope,
                      mask_zero=False,
                      trainable=False)
    a_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=ANSWER_LEN,
                      name="embedding_a_" + scope,
                      mask_zero=False,
                      trainable=False)
    c_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=CONTEXT_LEN,
                      name="embedding_c_" + scope,
                      mask_zero=False,
                      trainable=False)

    q = q_emb(q_input)
    a = a_emb(a_input)
    c = c_emb(c_input)

    q_lstm = LSTM(150, recurrent_dropout=0.15)(q)
    a_lstm = LSTM(150, recurrent_dropout=0.15)(a)
    c_lstm = LSTM(150, recurrent_dropout=0.15)(c)

    cqa = concatenate([c_lstm, q_lstm, a_lstm], axis=1)
    cqa = Dropout(0.25)(cqa)

    output_1 = Dense(250, activation='relu')(cqa)
    output_1 = Dropout(0.25)(output_1)
    output_2 = Dense(350, activation='relu')(output_1)

    output = Dense(2, activation='softmax')(output_2)
    model = Model(inputs=[q_input,
                          a_input, c_input], outputs=[output])
    model.compile(loss=categorical_crossentropy,
                  optimizer='adam',
                  metrics=['accuracy'])
    plot_model(model, to_file='2way_model.png', show_shapes=True)
    return model 

def attention_heatmap(num_words, embeddings_matrix, scope, embedding_dim=64):
    # (batch, input_len) => (batch, input_len, embedding_dim)
    q_input = Input(shape=(QUESTION_LEN,), name="q_input")
    a_input = Input(shape=(ANSWER_LEN,), name="a_input")
    c_input = Input(shape=(CONTEXT_LEN,), name="c_input")

    q_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=QUESTION_LEN,
                      name="embedding_q_" + scope,
                      mask_zero=False,
                      trainable=False)
    a_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=ANSWER_LEN,
                      name="embedding_a_" + scope,
                      mask_zero=False,
                      trainable=False)
    c_emb = Embedding(input_dim=num_words + 1,  # word 0 used for padding
                      output_dim=embedding_dim,
                      weights=[embeddings_matrix],
                      input_length=CONTEXT_LEN,
                      name="embedding_c_" + scope,
                      mask_zero=False,
                      trainable=False)

    q = q_emb(q_input)
    a = a_emb(a_input)
    c = c_emb(c_input)

    q = TimeDistributed(Dense(300, activation='tanh'))(q)
    a = TimeDistributed(Dense(300, activation='tanh'))(a)
    c = TimeDistributed(Dense(300, activation='tanh'))(c)

    q_lstm = Bidirectional(LSTM(50, recurrent_dropout=0.35))(q)
    c_lstm = Bidirectional(LSTM(50, recurrent_dropout=0.35,
                                return_sequences=True))(c)

    aux1 = TimeDistributed(Dense(200, activation=None,
                                 use_bias=False))(c_lstm)

    aux2 = Dense(200, activation=None, use_bias=False)(q_lstm)
    aux2 = RepeatVector(CONTEXT_LEN)(aux2)

    mt = Add()([aux1, aux2])
    mt = TimeDistributed(Activation('tanh'))(mt)

    st = TimeDistributed(Dense(1, activation=None, use_bias=False))(mt)
    st = Reshape((CONTEXT_LEN,))(st)
    st = Activation('softmax')(st)
    st = Reshape((CONTEXT_LEN, 1))(st)

    model = Model(inputs=[q_input, a_input, c_input], outputs=[st])
    model.compile(loss=categorical_crossentropy,
                  optimizer='adam',
                  metrics=['accuracy'])
    return model 

def define_model(self, embeddings_matrix, scope):
        assert(scope in ["train", "test"])
        assert(embeddings_matrix.shape[1] == WORD_EMBEDDINGS_DIM)

        tf_idf_score_input = Input(shape=(1,), name="tf_idf_score_input")
        second_best_tf_idf_input = Input(shape=(1,),
                                         name="second_best_tf_idf_input")
        question_input = Input(shape=(QUESTION_LEN,), name="question_input")
        qa_score_input = Input(shape=(2,), name="qa_score_input")
        scitail_score_input = Input(shape=(2,), name="scitail_score_input")
        snli_score_input = Input(shape=(2,), name="snli_score_input")
        multinli_score_input = Input(shape=(2,), name="multinli_score_input")
        qtype_input = Input(shape=(37,), name="qtype_input")

        question = Embedding(input_dim=embeddings_matrix.shape[0],
                             output_dim=embeddings_matrix.shape[1],
                             weights=[embeddings_matrix],
                             input_length=QUESTION_LEN,
                             name="embedding_q_" + scope,
                             mask_zero=True,
                             trainable=False)(question_input)
        question = LSTM(10, recurrent_dropout=0.5, name="lstm_1")(question)
        question = Dropout(0.5, name="dropout_1")(question)

        inputs = Concatenate(axis=-1, name="concatenate_1")([
                    tf_idf_score_input,
                    second_best_tf_idf_input,
                    qa_score_input,
                    scitail_score_input,
                    snli_score_input,
                    multinli_score_input,
                    # qtype_input
                    # question,
        ])
        inputs = Dropout(0.4)(inputs)

        output_1 = Dense(5, activation='relu', name="dense_1")(inputs)
        output_1 = Dropout(0.4, name="dropout_2")(output_1)

        output_2 = Dense(20, activation='relu', name="dense_2")(output_1)
        output_2 = Dropout(0.35, name="dropout_3")(output_2)

        output = Dense(NUM_CLASSES, activation='softmax', name="dense_5")(
                                                                    output_2)
        model = Model(inputs=[
                        tf_idf_score_input,
                        second_best_tf_idf_input,
                        question_input,
                        qa_score_input,
                        scitail_score_input,
                        snli_score_input,
                        multinli_score_input,
                        qtype_input
                      ], outputs=[output])
        model.compile(loss=categorical_crossentropy,
                      optimizer='adam',
                      metrics=['accuracy'])
        return model 

def embedding_mlp(num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks,
                  num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset,
                  units=FLAGS.units, dropout=FLAGS.dropout):
    import keras
    from keras.layers import Dense, Dropout
    from keras.losses import categorical_crossentropy
    from keras.callbacks import EarlyStopping
    from keras import backend as K

    if dataset != 'celeba':
        _, _, _, X_test, Y_test, Z_test = get_data(dataset, num_encoding_dims, test_set)
        task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots+5)
        partition = task_generator.get_partition_from_labels(Y_test)
        partitions = [partition]
    else:
        _, _, _, X_test, attributes_test, Z_test = get_data(dataset, num_encoding_dims, test_set)
        task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots+5)
        partitions = task_generator.get_celeba_task_pool(attributes_test)
    tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions)

    train_accuracies, test_accuracies = [], []

    start = time.time()
    for i_task, task in enumerate(tqdm(tasks)):
        if (i_task + 1) % (num_tasks // 10) == 0:
            tqdm.write('test {}, accuracy {:.5}'.format(i_task + 1, np.mean(test_accuracies)))
        ind_train_few, Y_train_few, ind_test_few, Y_test_few = task
        Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ind_test_few]
        Y_train_few, Y_test_few = keras.utils.to_categorical(Y_train_few, num_classes=num_classes), keras.utils.to_categorical(Y_test_few, num_classes=num_classes)

        model = keras.Sequential()
        model.add(Dense(units=units, activation='relu', input_dim=Z_train_few.shape[1]))
        model.add(Dropout(rate=dropout))
        model.add(Dense(units=num_classes, activation='softmax'))
        model.compile(loss=categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
        early_stopping = EarlyStopping(monitor='val_loss', patience=2)
        model.fit(Z_train_few, Y_train_few, batch_size=Z_train_few.shape[0], epochs=500, verbose=0, validation_data=(Z_test_few, Y_test_few), callbacks=[early_stopping])
        train_score = model.evaluate(Z_train_few, Y_train_few, verbose=0)
        train_accuracies.append(train_score[1])
        test_score = model.evaluate(Z_test_few, Y_test_few, verbose=0)
        test_accuracies.append(test_score[1])
        K.clear_session()

    print('units={}, dropout={}'.format(units, dropout))
    print('{}-way {}-shot embedding mlp: {:.5} with 95% CI {:.5} over {} tests'.format(num_classes, num_shots, np.mean(test_accuracies), 1.96*np.std(test_accuracies)/np.sqrt(num_tasks), num_tasks))
    print('Mean training accuracy: {:.5}; standard deviation: {:.5}'.format(np.mean(train_accuracies), np.std(train_accuracies)))
    print('{} few-shot classification tasks: {:.5} seconds.'.format(num_tasks, time.time() - start)) 

def standard_all(model_, model_name, big_X_train, big_y_train,
                 big_X_test, big_y_test, ch_num, class_names=class_names, addon=''):
    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model = EEGNet_org(nb_classes=len(class_names), Chans=ch_num, Samples=1125, dropoutRate=0.2)

        model.compile(loss=categorical_crossentropy,
                      optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, 'standard_user_{}'.format(i + 1), addon)
        metrics[i] = standard_unit(model, features_train[i], labels_train[i],
                                   features_test[i], labels_test[i], filename_,
                                   class_names)

    metrics_to_csv(metrics, '{}_Standard_testing_{}'.format(model_name, addon)) 

def full_distributed(model_name, big_X_train, big_y_train, big_X_test, big_y_test,
                     class_names=class_names, ch_num=25, dr=0.1, addon=''):

    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    # Flatten the data for training
    full_features = np.vstack(features_train)
    full_labels = np.vstack(labels_train)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model = EEGNet_org(nb_classes=len(class_names), Chans=ch_num, Samples=1125, dropoutRate=0.2)

        model.compile(loss=categorical_crossentropy,
                      optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, 'Distributed_{}'.format(i + 1), addon)
        metrics[i] = distributed_unit(model, full_features, full_labels,
                                      features_test[i], labels_test[i],
                                      filename_, class_names)

    metrics_to_csv(metrics, '{}_Distributed_Learning_{}'.format(model_name, filename_)) 

def full_freezing(model_name, big_X_train, big_y_train, big_X_test, big_y_test,
                    class_names=class_names, ch_num=25, dr=0.1, addon=''):
    ###
    ### Take all the data for training
    ###

    ###
    ### Train and save the model
    ###

    my_list = [0, 1, 2, 3, 4, 5, 6, 7, 8]

    model = EEGNet_org(nb_classes=2, Chans=ch_num, Samples=1125, dropoutRate=dr)

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    features_train = []
    labels_train = []

    for i in my_list:
        temp = [item for sublist in big_X_train[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_train[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Also add the testing data to increase the size of training data
        temp = [item for sublist in big_X_test[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_test[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Flatten the data for training
        full_features = np.vstack(features_train)
        full_labels = np.vstack(labels_train)

    filename_ = '{0}{1}{2}'.format(model_name, '_Full_Freezing', addon)

    full_freezing_unit(model, full_features, full_labels,
                       filename_, class_names) 

def standard_4mvt_2mvt(model_name, model_file, big_X_train, big_y_train, big_X_test, big_y_test,
                     class_names=class_names, ch_num=25, ep=50, dr=0.1, addon=''):
    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model_ = EEGNet_var(model_file, 4, 2, Chans=ch_num, dropoutRate=dr, Samples=1125)

        model_.compile(loss=categorical_crossentropy,
                       optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, '_transfer_standard_learning_{}_'.format(i + 1), addon)
        metrics[i] = transfer_unit(model_, features_train[i], labels_train[i],
                                   features_test[i], labels_test[i], filename_,
                                   class_names)

    metrics_to_csv(metrics, '{}_transfer_standard_learning_{}'.format(model_name, addon)) 

def full_distributed(model_name, big_X_train, big_y_train, big_X_test, big_y_test,
                     class_names=class_names, ch_num=25, dr=0.1, addon=''):

    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    # Flatten the data for training
    full_features = np.vstack(features_train)
    full_labels = np.vstack(labels_train)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model = EEGNet_org(nb_classes=len(class_names), Chans=ch_num, Samples=1125, dropoutRate=0.2)

        model.compile(loss=categorical_crossentropy,
                      optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, 'Distributed_{}'.format(i + 1), addon)
        metrics[i] = distributed_unit(model, full_features, full_labels,
                                      features_test[i], labels_test[i],
                                      filename_, class_names)

    metrics_to_csv(metrics, '{}_Distributed_Learning_{}'.format(model_name, filename_)) 

def full_freezing(model_name, big_X_train, big_y_train, big_X_test, big_y_test,
                    class_names=class_names, ch_num=25, dr=0.1, addon=''):
    ###
    ### Take all the data for training
    ###

    ###
    ### Train and save the model
    ###

    my_list = [0, 1, 2, 3, 4, 5, 6, 7]

    model = EEGNet_org(nb_classes=4, Chans=ch_num, Samples=1125, dropoutRate=dr)

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    features_train = []
    labels_train = []

    for i in my_list:
        temp = [item for sublist in big_X_train[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_train[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Also add the testing data to increase the size of training data
        temp = [item for sublist in big_X_test[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_test[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Flatten the data for training
        full_features = np.vstack(features_train)
        full_labels = np.vstack(labels_train)

    filename_ = '{0}{1}{2}'.format(model_name, '_Full_Freezing', addon)

    full_freezing_unit(model, full_features, full_labels,
                       filename_, class_names) 

def standard_from_2mvt(model_name, model_file, big_X_train, big_y_train, big_X_test, big_y_test,
                     class_names=class_names, ch_num=25, ep=50, dr=0.1, addon=''):
    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model_ = EEGNet_var(model_file, 2, 4, Chans=ch_num, dropoutRate=dr, Samples=1125)

        model_.compile(loss=categorical_crossentropy,
                      optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, '_transfer_standard_learning_{}_'.format(i + 1), addon)
        metrics[i] = transfer_unit(model_, features_train[i], labels_train[i],
                                   features_test[i], labels_test[i], filename_,
                                   class_names)

    metrics_to_csv(metrics, '{}_transfer_standard_learning_{}'.format(model_name, addon)) 

def standard_all(model_, model_name, big_X_train, big_y_train,
                 big_X_test, big_y_test, ch_num, class_names=class_names, addon=''):
    features_train = [None] * len(big_X_train)
    labels_train = [None] * len(big_y_train)

    for i, (X_user, y_user) in enumerate(zip(big_X_train, big_y_train)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train[i] = np_utils.to_categorical(encoded_Y)

    features_test = [None] * len(big_X_test)
    labels_test = [None] * len(big_y_test)
    for i, (X_user, y_user) in enumerate(zip(big_X_test, big_y_test)):
        temp = [item for sublist in X_user for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_test[i] = temp.reshape(temp.shape[0], 1, ch_num, 1125)
        lab = [item for sublist in y_user for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_test[i] = np_utils.to_categorical(encoded_Y)

    metrics = np.zeros(((len(big_y_train)), 4))

    for i in range(len(big_X_train)):
        model = EEGNet_org(nb_classes=len(class_names), Chans=ch_num, Samples=1125, dropoutRate=0.2)

        model.compile(loss=categorical_crossentropy,
                      optimizer=Adam(), metrics=['accuracy'])

        filename_ = '{0}{1}{2}'.format(model_name, 'standard_user_{}'.format(i + 1), addon)
        metrics[i] = standard_unit(model, features_train[i], labels_train[i],
                                   features_test[i], labels_test[i], filename_,
                                   class_names)

    metrics_to_csv(metrics, '{}_Standard_testing_{}'.format(model_name, addon)) 

def full_freezing(model_name, big_X_train, big_y_train, big_X_test, big_y_test,
                  class_names=class_names, ch_num=25, dr=0.1, addon=''):
    ###
    ### Take all the data for training
    ###

    ###
    ### Train and save the model
    ###

    my_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]

    model = EEGNet_org(nb_classes=2, Chans=ch_num, Samples=1125, dropoutRate=dr)

    model.compile(loss=categorical_crossentropy,
                  optimizer=Adam(), metrics=['accuracy'])

    features_train = []
    labels_train = []

    for i in my_list:
        temp = [item for sublist in big_X_train[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_train[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Also add the testing data to increase the size of training data
        temp = [item for sublist in big_X_test[i] for item in sublist]
        temp = np.asanyarray(temp)
        temp = np.swapaxes(temp, 1, 2)
        features_train.append(temp.reshape(temp.shape[0], 1, ch_num, 1125))
        lab = [item for sublist in big_y_test[i] for item in sublist]
        # encode class values as integers
        encoder = LabelEncoder()
        encoder.fit(lab)
        encoded_Y = encoder.transform(lab)
        # convert integers to dummy variables (i.e. one hot encoded)
        labels_train.append(np_utils.to_categorical(encoded_Y))

        # Flatten the data for training
        full_features = np.vstack(features_train)
        full_labels = np.vstack(labels_train)

    filename_ = '{0}{1}{2}'.format(model_name, '_Full_Freezing_2mvt', addon)

    full_freezing_unit(model, full_features, full_labels,
                       filename_, class_names) 

def build_model(self, input_shape, n_classes):
        ## input layer
        input_layer = Input(input_shape)

        ## convolutional layers
        conv_layer1 = Conv2D(filters=8, kernel_size=(3, 3), activation='relu')(input_layer)
        conv_layer2 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu')(conv_layer1)

        ## add max pooling to obtain the most imformatic features
        pooling_layer1 = MaxPool2D(pool_size=(2, 2))(conv_layer2)

        conv_layer3 = Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(pooling_layer1)
        conv_layer4 = Conv2D(filters=64, kernel_size=(3, 3), activation='relu')(conv_layer3)
        pooling_layer2 = MaxPool2D(pool_size=(2, 2))(conv_layer4)

        ## perform batch normalization on the convolution outputs before feeding it to MLP architecture
        pooling_layer2 = BatchNormalization()(pooling_layer2)
        flatten_layer = Flatten()(pooling_layer2)

        ## create an MLP architecture with dense layers : 4096 -> 512 -> 10
        ## add dropouts to avoid overfitting / perform regularization
        dense_layer1 = Dense(units=2048, activation='relu')(flatten_layer)
        dense_layer1 = Dropout(0.4)(dense_layer1)
        dense_layer2 = Dense(units=512, activation='relu')(dense_layer1)
        dense_layer2 = Dropout(0.4)(dense_layer2)
        output_layer = Dense(units=n_classes, activation='softmax')(dense_layer2)

        ## define the model with input layer and output layer
        model = Model(inputs=input_layer, outputs=output_layer)
        model.summary()

        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)

        model.compile(loss=categorical_crossentropy, optimizer=Adadelta(), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        xception_1 = self.xception(include_top=False, pooling='max')
        for layer in xception_1.layers:
            layer.trainable = False
        input_1 = xception_1.input
        output_1 = xception_1.output
        xception_2 = self.xception(include_top=False, pooling='max')
        for layer in xception_2.layers:
            layer.trainable = False
        input_2 = xception_2.input
        output_2 = xception_2.output
        xception_3 = self.xception(include_top=False, pooling='max')
        for layer in xception_3.layers:
            layer.trainable = False
        input_3 = xception_3.input
        output_3 = xception_3.output
        xception_4 = self.xception(include_top=False, pooling='max')
        for layer in xception_4.layers:
            layer.trainable = False
        input_4 = xception_4.input
        output_4 = xception_4.output

        concat_layer = Maximum()([output_1, output_2, output_3, output_4])
        concat_layer.trainable = False
        # concat_layer = Dropout(0.25)(concat_layer)
        # dense_layer1 = Dense(units=1024, activation='relu')(concat_layer)
        dense_layer1 = Dropout(0.5)(concat_layer)
        output_layer = Dense(n_classes, activation='softmax', name='predictions')(dense_layer1)

        model = Model(inputs=[input_1, input_2, input_3, input_4], outputs=[output_layer])
        model.summary()
        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)
        model.compile(loss=categorical_crossentropy, optimizer=Adam(lr=0.01), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        # input layer
        input_layer = Input(input_shape)
        # Block 1
        x = Conv3D(64, (3, 3, 3), activation='relu', padding='same', name='block1_conv1')(input_layer)
        x = Conv3D(64, (3, 3, 3), activation='relu', padding='same', name='block1_conv2')(x)
        x = MaxPool3D((2, 2, 2), strides=(2, 2, 1), name='block1_pool')(x)

        # Block 2
        x = Conv3D(128, (3, 3, 3), activation='relu', padding='same', name='block2_conv1')(x)
        x = Conv3D(128, (3, 3, 3), activation='relu', padding='same', name='block2_conv2')(x)
        x = MaxPool3D((2, 2, 2), strides=(2, 2, 1), name='block2_pool')(x)

        # Block 3
        x = Conv3D(256, (3, 3, 2), activation='relu', padding='same', name='block3_conv1')(x)
        x = Conv3D(256, (3, 3, 2), activation='relu', padding='same', name='block3_conv2')(x)
        x = Conv3D(256, (3, 3, 2), activation='relu', padding='same', name='block3_conv3')(x)
        x = Conv3D(256, (3, 3, 2), activation='relu', padding='same', name='block3_conv4')(x)
        x = MaxPool3D((2, 2, 2), strides=(2, 2, 1), name='block3_pool')(x)

        # Block 4
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block4_conv1')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block4_conv2')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block4_conv3')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block4_conv4')(x)
        x = MaxPool3D((2, 2, 1), strides=(2, 2, 1), name='block4_pool')(x)

        # Block 5
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block5_conv1')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block5_conv2')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block5_conv3')(x)
        x = Conv3D(512, (3, 3, 1), activation='relu', padding='same', name='block5_conv4')(x)
        x = MaxPool3D((2, 2, 1), strides=(2, 2, 1), name='block5_pool')(x)

        # Classification block
        x = Flatten(name='flatten')(x)
        x = Dense(4096, activation='relu', name='fc1')(x)
        x = Dropout(0.4)(x)
        x = Dense(4096, activation='relu', name='fc2')(x)
        x = Dropout(0.4)(x)
        output_layer = Dense(n_classes, activation='softmax', name='predictions')(x)

        ## define the model with input layer and output layer
        model = Model(inputs=input_layer, outputs=output_layer)
        model.summary()

        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)

        model.compile(loss=categorical_crossentropy, optimizer=Adadelta(lr=0.1), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        ## input layer
        input_layer = Input(input_shape)

        ## convolutional layers
        conv_layer1 = Conv3D(filters=8, kernel_size=(3, 3, 2), activation='relu')(input_layer)
        conv_layer2 = Conv3D(filters=16, kernel_size=(3, 3, 2), activation='relu')(conv_layer1)

        ## add max pooling to obtain the most imformatic features
        pooling_layer1 = MaxPool3D(pool_size=(2, 2, 2))(conv_layer2)

        conv_layer3 = Conv3D(filters=32, kernel_size=(3, 3, 1), activation='relu')(pooling_layer1)
        conv_layer4 = Conv3D(filters=64, kernel_size=(3, 3, 1), activation='relu')(conv_layer3)
        pooling_layer2 = MaxPool3D(pool_size=(2, 2, 1))(conv_layer4)

        ## perform batch normalization on the convolution outputs before feeding it to MLP architecture
        pooling_layer2 = BatchNormalization()(pooling_layer2)
        flatten_layer = Flatten()(pooling_layer2)

        ## create an MLP architecture with dense layers : 4096 -> 512 -> 10
        ## add dropouts to avoid overfitting / perform regularization
        dense_layer1 = Dense(units=2048, activation='relu')(flatten_layer)
        dense_layer1 = Dropout(0.4)(dense_layer1)
        dense_layer2 = Dense(units=512, activation='relu')(dense_layer1)
        dense_layer2 = Dropout(0.4)(dense_layer2)
        output_layer = Dense(units=n_classes, activation='softmax')(dense_layer2)

        ## define the model with input layer and output layer
        model = Model(inputs=input_layer, outputs=output_layer)
        model.summary()

        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)

        model.compile(loss=categorical_crossentropy, optimizer=Adadelta(), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model 

def build_model(self, input_shape, n_classes):
        # input layer
        input_layer = Input(input_shape)
        # Block 1
        x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(input_layer)
        x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
        x = MaxPool2D((2, 2), strides=(2, 2), name='block1_pool')(x)

        # Block 2
        x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
        x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
        x = MaxPool2D((2, 2), strides=(2, 2), name='block2_pool')(x)

        # Block 3
        x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
        x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
        x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
        x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv4')(x)
        x = MaxPool2D((2, 2), strides=(2, 2), name='block3_pool')(x)

        # Block 4
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv4')(x)
        x = MaxPool2D((2, 2), strides=(2, 2), name='block4_pool')(x)

        # Block 5
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
        x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv4')(x)
        x = MaxPool2D((2, 2), strides=(2, 2), name='block5_pool')(x)

        # Classification block
        x = Flatten(name='flatten')(x)
        x = Dense(4096, activation='relu', name='fc1')(x)
        x = Dense(4096, activation='relu', name='fc2')(x)
        output_layer = Dense(n_classes, activation='softmax', name='predictions')(x)

        ## define the model with input layer and output layer
        model = Model(inputs=input_layer, outputs=output_layer)
        model.summary()

        plot_model(model, to_file=self.output_directory + '/model_graph.png', show_shapes=True, show_layer_names=True)

        model.compile(loss=categorical_crossentropy, optimizer=Adadelta(lr=0.1), metrics=['acc'])

        # model save
        file_path = self.output_directory + '/best_model.hdf5'
        model_checkpoint = callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True)

        # Tensorboard log
        log_dir = self.output_directory + '/tf_logs'
        chk_n_mkdir(log_dir)
        tb_cb = TrainValTensorBoard(log_dir=log_dir)
        
        self.callbacks = [model_checkpoint, tb_cb]
        return model