Python keras.activations.softmax() Examples

The following are code examples for showing how to use keras.activations.softmax(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: Jtyoui   Author: jtyoui   File: HandWritingRecognition.py    MIT License 6 votes vote down vote up
def nn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train.reshape(x_train.shape[0], -1) / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)
    # constant(value=1.)自定义常数,constant(value=1.)===one()
    # 创建模型:输入784个神经元,输出10个神经元
    model = Sequential([
        Dense(units=200, input_dim=784, bias_initializer=constant(value=1.), activation=tanh),
        Dense(units=100, bias_initializer=one(), activation=tanh),
        Dense(units=10, bias_initializer=one(), activation=softmax),
    ])

    opt = SGD(lr=0.2, clipnorm=1.)  # 优化器
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['acc', 'mae'])  # 编译
    model.fit(x_train, y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 
Example 2
Project: CAPTCHA-breaking   Author: lllcho   File: test_activations.py    MIT License 6 votes vote down vote up
def test_softmax():

    from keras.activations import softmax as s

    # Test using a reference implementation of softmax
    def softmax(values):
        m = max(values)
        values = numpy.array(values)
        e = numpy.exp(values - m)
        dist = list(e / numpy.sum(e))

        return dist

    x = T.vector()
    exp = s(x)
    f = theano.function([x], exp)
    test_values=get_standard_values()

    result = f(test_values)
    expected = softmax(test_values)

    print(str(result))
    print(str(expected))

    list_assert_equal(result, expected) 
Example 3
Project: spektral   Author: danielegrattarola   File: pooling.py    MIT License 6 votes vote down vote up
def call(self, inputs):
        if self.data_mode == 'graph':
            X, I = inputs
            if K.ndim(I) == 2:
                I = I[:, 0]
        else:
            X = inputs
        attn_coeff = K.dot(X, self.attn_kernel)
        attn_coeff = K.squeeze(attn_coeff, -1)
        attn_coeff = K.softmax(attn_coeff)
        if self.data_mode == 'single':
            output = K.dot(attn_coeff[None, ...], X)
        elif self.data_mode == 'batch':
            output = K.batch_dot(attn_coeff, X)
        else:
            output = attn_coeff[:, None] * X
            output = tf.segment_sum(output, I)

        return output 
Example 4
Project: pointer-networks-experiments   Author: zygmuntz   File: PointerLSTM.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def step(self, x_input, states):
    	#print "x_input:", x_input, x_input.shape
    	# <TensorType(float32, matrix)>
    	
        input_shape = self.input_spec[0].shape
        en_seq = states[-1]
        _, [h, c] = super(PointerLSTM, self).step(x_input, states[:-1])

        # vt*tanh(W1*e+W2*d)
        dec_seq = K.repeat(h, input_shape[1])
        Eij = time_distributed_dense(en_seq, self.W1, output_dim=1)
        Dij = time_distributed_dense(dec_seq, self.W2, output_dim=1)
        U = self.vt * tanh(Eij + Dij)
        U = K.squeeze(U, 2)

        # make probability tensor
        pointer = softmax(U)
        return pointer, [h, c] 
Example 5
Project: workspace_2017   Author: nwiizo   File: test_activations.py    MIT License 6 votes vote down vote up
def test_softmax():
    '''
    Test using a reference implementation of softmax
    '''
    def softmax(values):
        m = np.max(values)
        e = np.exp(values - m)
        return e / np.sum(e)

    x = K.placeholder(ndim=2)
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = softmax(test_values)
    assert_allclose(result, expected, rtol=1e-05) 
Example 6
Project: keras-yolo   Author: BrainsGarden   File: region.py    MIT License 6 votes vote down vote up
def __init__(self, coords=4, classes=20, num=1,
            log=0, sqrt=0, softmax=0, background=0, max=30,
            jitter=0.2, 
            rescore = 0, thresh=0.5, classfix=0, absolute=0, random=0,
            coord_scale=1, object_scale=1,
            noobject_scale=1, class_scale=1,
            bias_match=0,
            tree=None,#tree_file for softmax_tree - not used now
            map_filename=None, # file name for map_file - not used
            anchors=None,
            **kwargs
            ):
        super(Region, self).__init__(**kwargs)
        self.coords = coords
        self.classes = classes
        self.num = num
        self.background = background
        print(coords, classes)
        self.c = (self.coords+self.classes+1)*num
        if anchors:
            self.biases = list(map(float, anchors))
        pass 
Example 7
Project: keras-yolo   Author: BrainsGarden   File: region.py    MIT License 6 votes vote down vote up
def _process_input(self, x):
        """Apply logistic and softmax activations to input tensor
        """
        logistic_activate = lambda x: 1.0/(1.0 + K.exp(-x))
        
        (batch, w, h, channels) = x.get_shape()
        x_temp = K.permute_dimensions(x, (3, 0, 1, 2))
        x_t = []
        for i in range(self.num):
            k = self._entry_index(i, 0)
            x_t.extend([
                logistic_activate(K.gather(x_temp, (k, k + 1))), # 0
                K.gather(x_temp, (k + 2, k + 3))])
            if self.background:
                x_t.append(K.gather(x_temp, (k + 4,)))
            else:
                x_t.append(logistic_activate(K.gather(x_temp, (k + 4,))))
                
            x_t.append(
                softmax(
                    K.gather(x_temp, tuple(range(k + 5, k + self.coords + self.classes + 1))),
                    axis=0))
        x_t = K.concatenate(x_t, axis=0)
        return K.permute_dimensions(x_t, (1, 2, 3, 0)) 
Example 8
Project: ntm_keras   Author: flomlo   File: ntm.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _get_weight_vector(self, M, w_tm1, k, beta, g, s, gamma):
#        M = tf.Print(M, [M, w_tm1, k], message='get weights beg1: ')
#        M = tf.Print(M, [beta, g, s, gamma], message='get weights beg2: ')
        # Content adressing, see Chapter 3.3.1:
        num = beta * _cosine_distance(M, k)
        w_c  = K.softmax(num) # It turns out that equation (5) is just softmax.
        # Location adressing, see Chapter 3.3.2:
        # Equation 7:
        w_g = (g * w_c) + (1-g)*w_tm1
        # C_s is the circular convolution
        #C_w = K.sum((self.C[None, :, :, :] * w_g[:, None, None, :]),axis=3)
        # Equation 8:
        # TODO: Explain
        C_s = K.sum(K.repeat_elements(self.C[None, :, :, :], self.batch_size, axis=0) * s[:,:,None,None], axis=1)
        w_tilda = K.batch_dot(C_s, w_g)
        # Equation 9:
        w_out = _renorm(w_tilda ** gamma)

        return w_out 
Example 9
Project: CNNArt   Author: thomaskuestner   File: motion_all_CNN2D_multiscale.py    Apache License 2.0 6 votes vote down vote up
def createModel(patchSize, patchSize_down=None, ScaleFactor=1, learningRate=1e-3, optimizer='SGD',
                     dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    # Total params: 453,570
    input_orig = Input(shape=(1, int(patchSize[0]), int(patchSize[1])))
    path_orig_output = fConveBlock(input_orig)
    input_down = Input(shape=(1, int(patchSize_down[0]), int(patchSize_down[1])))
    path_down = fConveBlock(input_down)
    path_down_output = fUpSample(path_down, ScaleFactor)
    multi_scale_connect = fconcatenate(path_orig_output, path_down_output)

    # fully connect layer as dense
    flat_out = Flatten()(multi_scale_connect)
    dropout_out = Dropout(dr_rate)(flat_out)
    dense_out = Dense(units=2,
                          kernel_initializer='normal',
                          kernel_regularizer=l2(l2_reg))(dropout_out)
    # Fully connected layer as convo with 1X1 ?

    output_fc1 = Activation('softmax')(dense_out)
    output_fc2 = Activation('softmax')(dense_out)
    output_p1 = Lambda(sliceP1,name='path1_output',output_shape=(None,2))(output_fc1)
    output_p2 = Lambda(sliceP2,name='path2_output',output_shape=(None,2))(output_fc2)
    cnn_ms = Model(inputs=[input_orig, input_down], outputs=[output_p1,output_p2])
    return cnn_ms 
Example 10
Project: EEG_classification   Author: CVxTz   File: models.py    Apache License 2.0 6 votes vote down vote up
def get_model_lstm():
    nclass = 5

    seq_input = Input(shape=(None, 3000, 1))
    base_model = get_base_model()
    for layer in base_model.layers:
        layer.trainable = False
    encoded_sequence = TimeDistributed(base_model)(seq_input)
    encoded_sequence = Bidirectional(LSTM(100, return_sequences=True))(encoded_sequence)
    encoded_sequence = Dropout(rate=0.5)(encoded_sequence)
    encoded_sequence = Bidirectional(LSTM(100, return_sequences=True))(encoded_sequence)
    #out = TimeDistributed(Dense(nclass, activation="softmax"))(encoded_sequence)
    out = Convolution1D(nclass, kernel_size=1, activation="softmax", padding="same")(encoded_sequence)

    model = models.Model(seq_input, out)

    model.compile(optimizers.Adam(0.001), losses.sparse_categorical_crossentropy, metrics=['acc'])
    model.summary()

    return model 
Example 11
Project: talos   Author: autonomio   File: params.py    MIT License 6 votes vote down vote up
def iris():

    from keras.optimizers import Adam, Nadam
    from keras.losses import logcosh, categorical_crossentropy
    from keras.activations import relu, elu, softmax

    # here use a standard 2d dictionary for inputting the param boundaries
    p = {'lr': (0.5, 5, 10),
         'first_neuron': [4, 8, 16, 32, 64],
         'hidden_layers': [0, 1, 2, 3, 4],
         'batch_size': (2, 30, 10),
         'epochs': [2],
         'dropout': (0, 0.5, 5),
         'weight_regulizer': [None],
         'emb_output_dims':  [None],
         'shapes': ['brick', 'triangle', 0.2],
         'optimizer': [Adam, Nadam],
         'losses': [logcosh, categorical_crossentropy],
         'activation': [relu, elu],
         'last_activation': [softmax]}

    return p 
Example 12
Project: C3AE   Author: StevenBanama   File: C3AE.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def build_net(CATES=12, height=64, width=64, channel=3, using_white_norm=True, using_SE=True):
    base_model = build_shared_plain_network(using_white_norm=using_white_norm, using_SE=using_SE)
    print(base_model.summary())
    x1 = Input(shape=(height, width, channel))
    x2 = Input(shape=(height, width, channel))
    x3 = Input(shape=(height, width, channel))

    y1 = base_model(x1)
    y2 = base_model(x2)
    y3 = base_model(x3)

    cfeat = Concatenate(axis=-1)([y1, y2, y3])
    bulk_feat = Dense(CATES, use_bias=True, activity_regularizer=regularizers.l1(0), activation=softmax)(cfeat)
    age = Dense(1, name="age")(bulk_feat)
    #age = Lambda(lambda a: tf.reshape(tf.reduce_sum(a * tf.constant([[x * 10.0 for x in xrange(12)]]), axis=-1), shape=(-1, 1)), name="age")(bulk_feat)
    return Model(input=[x1, x2, x3], output=[age, bulk_feat]) 
Example 13
Project: Jtyoui   Author: jtyoui   File: HandWritingRecognition.py    MIT License 5 votes vote down vote up
def cnn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train.reshape(-1, 28, 28, 1) / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)

    model = Sequential([
        # input_shape:输入平面,就在第一个位置设置
        # filters:卷积核、滤波器
        # kernel_size:卷积核大小
        # strides:步长
        # padding有两种方式:same/valid
        # activation:激活函数
        Convolution2D(input_shape=(28, 28, 1), filters=32, kernel_size=5, strides=1, padding='same', activation=relu),
        MaxPool2D(pool_size=2, strides=2, padding='same'),
        Convolution2D(filters=64, kernel_size=5, padding='same', activation=relu),
        MaxPool2D(pool_size=2, trainable=2, padding='same'),
        Flatten(),  # 扁平化
        Dense(units=1024, activation=relu),
        Dropout(0.5),
        Dense(units=10, activation=softmax),
    ])
    opt = Adam(lr=1e-4)
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
    model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 
Example 14
Project: Jtyoui   Author: jtyoui   File: HandWritingRecognition.py    MIT License 5 votes vote down vote up
def rnn_model():
    (x_train, y_train), _ = mnist.load_data()
    # 归一化
    x_train = x_train / 255.
    # one-hot
    y_train = np_utils.to_categorical(y=y_train, num_classes=10)

    model = Sequential([
        SimpleRNN(units=50, input_shape=(28, 28)),
        Dense(units=10, activation=softmax),
    ])
    opt = RMSprop(lr=1e-4)
    model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
    model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
    model_save(model, './model.h5') 
Example 15
Project: workspace_2017   Author: nwiizo   File: test_activations.py    MIT License 5 votes vote down vote up
def test_time_distributed_softmax():
    x = K.placeholder(shape=(1, 1, 5))
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()
    test_values = np.reshape(test_values, (1, 1, np.size(test_values)))
    f([test_values])[0] 
Example 16
Project: foolbox   Author: bethgelab   File: test_models_keras.py    MIT License 5 votes vote down vote up
def test_keras_model_probs(num_classes):
    bounds = (0, 255)
    channels = num_classes

    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        inputs = Input(shape=(5, 5, channels))
        logits = GlobalAveragePooling2D(data_format="channels_last")(inputs)
        probs = Activation(softmax)(logits)

        model1 = KerasModel(
            Model(inputs=inputs, outputs=logits), bounds=bounds, predicts="logits"
        )

        model2 = KerasModel(
            Model(inputs=inputs, outputs=probs), bounds=bounds, predicts="probabilities"
        )

        model3 = KerasModel(
            Model(inputs=inputs, outputs=probs), bounds=bounds, predicts="probs"
        )

    np.random.seed(22)
    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)

    p1 = model1.forward(test_images)
    p2 = model2.forward(test_images)
    p3 = model3.forward(test_images)

    assert p1.shape == p2.shape == p3.shape == (2, num_classes)

    np.testing.assert_array_almost_equal(p1 - p1.max(), p2 - p2.max(), decimal=1)

    np.testing.assert_array_almost_equal(p2 - p2.max(), p3 - p3.max(), decimal=5) 
Example 17
Project: applications   Author: geomstats   File: activations_test.py    MIT License 5 votes vote down vote up
def test_serialization():
    all_activations = ['softmax', 'relu', 'elu', 'tanh',
                       'sigmoid', 'hard_sigmoid', 'linear',
                       'softplus', 'softsign', 'selu']
    for name in all_activations:
        fn = activations.get(name)
        ref_fn = getattr(activations, name)
        assert fn == ref_fn
        config = activations.serialize(fn)
        fn = activations.deserialize(config)
        assert fn == ref_fn 
Example 18
Project: applications   Author: geomstats   File: activations_test.py    MIT License 5 votes vote down vote up
def test_softmax_valid():
    """Test using a reference implementation of softmax.
    """
    def softmax(values):
        m = np.max(values)
        e = np.exp(values - m)
        return e / np.sum(e)

    x = K.placeholder(ndim=2)
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()

    result = f([test_values])[0]
    expected = softmax(test_values)
    assert_allclose(result, expected, rtol=1e-05) 
Example 19
Project: applications   Author: geomstats   File: activations_test.py    MIT License 5 votes vote down vote up
def test_softmax_invalid():
    """Test for the expected exception behaviour on invalid input
    """

    x = K.placeholder(ndim=1)

    # One dimensional arrays are supposed to raise a value error
    with pytest.raises(ValueError):
        f = K.function([x], [activations.softmax(x)]) 
Example 20
Project: applications   Author: geomstats   File: activations_test.py    MIT License 5 votes vote down vote up
def test_time_distributed_softmax():
    x = K.placeholder(shape=(1, 1, 5))
    f = K.function([x], [activations.softmax(x)])
    test_values = get_standard_values()
    test_values = np.reshape(test_values, (1, 1, np.size(test_values)))
    f([test_values])[0] 
Example 21
Project: MatchZoo   Author: NTMC-Community   File: anmm.py    Apache License 2.0 5 votes vote down vote up
def build(self):
        """
        Build model structure.

        aNMM model based on bin weighting and query term attentions
        """
        # query is [batch_size, left_text_len]
        # doc is [batch_size, right_text_len, bin_num]
        query, doc = self._make_inputs()
        embedding = self._make_embedding_layer()

        q_embed = embedding(query)
        q_attention = keras.layers.Dense(
            1, kernel_initializer=RandomUniform(), use_bias=False)(q_embed)
        q_text_len = self._params['input_shapes'][0][0]

        q_attention = keras.layers.Lambda(
            lambda x: softmax(x, axis=1),
            output_shape=(q_text_len,)
        )(q_attention)
        d_bin = keras.layers.Dropout(
            rate=self._params['dropout_rate'])(doc)
        for layer_id in range(self._params['num_layers'] - 1):
            d_bin = keras.layers.Dense(
                self._params['hidden_sizes'][layer_id],
                kernel_initializer=RandomUniform())(d_bin)
            d_bin = keras.layers.Activation('tanh')(d_bin)
        d_bin = keras.layers.Dense(
            self._params['hidden_sizes'][self._params['num_layers'] - 1])(
            d_bin)
        d_bin = keras.layers.Reshape((q_text_len,))(d_bin)
        q_attention = keras.layers.Reshape((q_text_len,))(q_attention)
        score = keras.layers.Dot(axes=[1, 1])([d_bin, q_attention])
        x_out = self._make_output_layer()(score)
        self._backend = keras.Model(inputs=[query, doc], outputs=x_out) 
Example 22
Project: digitx   Author: PowerOfDream   File: digitx_model.py    MIT License 5 votes vote down vote up
def digitx_model(input_shape):
    '''
    a model to classify printed 0-9 and x
    Arguments:
        input_shape -- shape of the input images
    
    Returns:
        model -- a Model() instance in Keras
    '''

    X_input = Input(input_shape, name = 'input')

    X = Conv2D(16, (5, 5), strides = (1, 1), padding = 'same', name = 'conv0')(X_input)
    X = BatchNormalization(axis = 3, name = 'bn0')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((2, 2), name = 'max_pool0')(X)

    X = Conv2D(32, (3, 3), strides = (1, 1), padding = 'same', name = 'conv1')(X)
    X = BatchNormalization(axis = 3, name = 'bn1')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((2, 2), name = 'max_pool1')(X)

    X = Conv2D(64, (3, 3), strides = (1, 1), padding = 'same', name = 'conv2')(X)
    X = BatchNormalization(axis = 3, name = 'bn2')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((2, 2), name = 'max_pool2')(X)

    X = Flatten()(X)
    X = BatchNormalization(axis = 1, name = 'bn3')(X)
    X = Dense(128, activation='relu', name = 'after_flatten')(X)
    X = Dense(64, activation='relu', name = 'after_flatten2')(X)
    X = Dense(11, activation='softmax', name = 'softmax_out', kernel_initializer = glorot_uniform())(X)

    model = Model(inputs = X_input, outputs = X, name = 'digitx_model')

    return model 
Example 23
Project: digitx   Author: PowerOfDream   File: digitx_model.py    MIT License 5 votes vote down vote up
def digitx_model2(input_shape):
    '''
    same as digitx_model, but use NIN & global average pooling
    '''

    X_input = Input(input_shape, name = 'input')

    X = Conv2D(32, (5, 5), strides = (1, 1), padding = 'same', name = 'conv0')(X_input)
    X = BatchNormalization(axis = 3, name = 'bn0')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((2, 2), name = 'max_pool0')(X)
    X = Conv2D(16, (1, 1), strides = (1, 1), padding = 'same', name = 'nin0')(X)

    X = Conv2D(32, (3, 3), strides = (1, 1), padding = 'same', name = 'conv1')(X)
    X = BatchNormalization(axis = 3, name = 'bn1')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((2, 2), name = 'max_pool1')(X)
    X = Conv2D(16, (1, 1), strides = (1, 1), padding = 'same', name = 'nin1')(X)

    X = Conv2D(32, (3, 3), strides = (1, 1), padding = 'same', name = 'conv2')(X)
    X = BatchNormalization(axis = 3, name = 'bn2')(X)
    X = Activation('relu')(X)

    # arXiv 1312.4400 : generate one feature map for each corresponding category.
    # Network in network, 1x1 Conv shrink the channel number into catagory number
    # output shape (None, 8, 8, 11), each fearure represented by a 8x8 matrix
    X = Conv2D(11, (1, 1), strides = (1, 1), padding = 'same', name = 'nin2')(X)

    # arXiv 1312.4400 : take the average of each feature map, 
    # fed the result vector directly into the softmax layer
    X = GlobalAveragePooling2D()(X)
    X = BatchNormalization(axis = 1, name = 'bn3')(X)
    X = Activation('softmax', name="softmax_out")(X)

    model = Model(inputs = X_input, outputs = X, name = 'digitx_model')

    return model 
Example 24
Project: CNNArt   Author: thomaskuestner   File: VNetArt.py    Apache License 2.0 5 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                 dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    l2_reg = 1e-4
    # using SGD lr 0.001
    # motion_head:unkorrigierte Version 3steps with only type(1,1,1)(149K params)--> val_loss: 0.2157 - val_acc: 0.9230
    # motion_head:korrigierte Version type(1,2,2)(266K params) --> val_loss: 0.2336 - val_acc: 0.9149 nach abbruch...
    # double_#channels(type 122) (870,882 params)>
    # functional api...
    input_t = Input(shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2])))

    after_res1_t = fCreateVNet_Block(input_t, 32, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv1_t = fCreateVNet_DownConv_Block(after_res1_t, after_res1_t._keras_shape[1], (2, 2, 2),
                                                   iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)

    after_res2_t = fCreateVNet_Block(after_DownConv1_t, 64, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv2_t = fCreateVNet_DownConv_Block(after_res2_t, after_res2_t._keras_shape[1], (2, 2, 1),
                                                   iPReLU=iPReLU, l2_reg=l2_reg, dr_rate=dr_rate)

    after_res3_t = fCreateVNet_Block(after_DownConv2_t, 128, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv3_t = fCreateVNet_DownConv_Block(after_res3_t, after_res3_t._keras_shape[1], (2, 2, 1),
                                                   iPReLU=iPReLU, l2_reg=l2_reg, dr_rate=dr_rate)

    after_flat_t = Flatten()(after_DownConv3_t)
    after_dense_t = Dropout(dr_rate)(after_flat_t)
    after_dense_t = Dense(units=2,
                          kernel_initializer='normal',
                          kernel_regularizer=l2(l2_reg))(after_dense_t)
    output_t = Activation('softmax')(after_dense_t)

    cnn = Model(inputs=[input_t], outputs=[output_t])

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)  # loss cat_crosent default
    cnn.compile(optimizer=opti, loss=loss, metrics=['accuracy'])
    sArchiSpecs = '_t222_l2{}_dr{}'.format(l2_reg, dr_rate) 
Example 25
Project: CNNArt   Author: thomaskuestner   File: motion_VNetArt.py    Apache License 2.0 5 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                     dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    l2_reg = 1e-4
    # using SGD lr 0.001
    # motion_head:unkorrigierte Version 3steps with only type(1,1,1)(149K params)--> val_loss: 0.2157 - val_acc: 0.9230
    # motion_head:korrigierte Version type(1,2,2)(266K params) --> val_loss: 0.2336 - val_acc: 0.9149 nach abbruch...
    # double_#channels(type 122) (870,882 params)>
    # functional api...
    input_t = Input(shape=(1, int(patchSize[0]), int(patchSize[1]), int(patchSize[2])))

    after_res1_t = fCreateVNet_Block(input_t, 32, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv1_t = fCreateVNet_DownConv_Block(after_res1_t, after_res1_t._keras_shape[1], (2, 2, 2),
                                                     iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)

    after_res2_t = fCreateVNet_Block(after_DownConv1_t, 64, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv2_t = fCreateVNet_DownConv_Block(after_res2_t, after_res2_t._keras_shape[1], (2, 2, 1),
                                                     iPReLU=iPReLU, l2_reg=l2_reg, dr_rate=dr_rate)

    after_res3_t = fCreateVNet_Block(after_DownConv2_t, 128, type=2, iPReLU=iPReLU, dr_rate=dr_rate, l2_reg=l2_reg)
    after_DownConv3_t = fCreateVNet_DownConv_Block(after_res3_t, after_res3_t._keras_shape[1], (2, 2, 1),
                                                     iPReLU=iPReLU, l2_reg=l2_reg, dr_rate=dr_rate)

    after_flat_t = Flatten()(after_DownConv3_t)
    after_dense_t = Dropout(dr_rate)(after_flat_t)
    after_dense_t = Dense(units=2,
                          kernel_initializer='normal',
                          kernel_regularizer=l2(l2_reg))(after_dense_t)
    output_t = Activation('softmax')(after_dense_t)

    cnn = Model(inputs=[input_t], outputs=[output_t])

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)  # loss cat_crosent default
    cnn.compile(optimizer=opti, loss=loss, metrics=['accuracy'])
    sArchiSpecs = '_t222_l2{}_dr{}'.format(l2_reg, dr_rate)
    return cnn 
Example 26
Project: EvadeML-Zoo   Author: mzweilin   File: magnet_cifar.py    MIT License 5 votes vote down vote up
def __init__(self, model):
        """
        Keras classifier wrapper.
        Note that the wrapped classifier should spit logits as output.
        """
        layer_id = len(model.layers)-2
        self.model = Model(inputs=model.layers[0].input, outputs=model.layers[layer_id].output)
        self.softmax = Sequential()
        self.softmax.add(Lambda(lambda X: softmax(X, axis=1), input_shape=(10,))) 
Example 27
Project: EvadeML-Zoo   Author: mzweilin   File: magnet_cifar.py    MIT License 5 votes vote down vote up
def classify(self, X, option="logit", T=1):
        if option == "logit":
            return self.model.predict(X)
        if option == "prob":
            logits = self.model.predict(X)/T
            return self.softmax.predict(logits) 
Example 28
Project: EvadeML-Zoo   Author: mzweilin   File: magnet_mnist.py    MIT License 5 votes vote down vote up
def __init__(self, model):
        """
        Keras classifier wrapper.
        Note that the wrapped classifier should spit logits as output.
        """
        layer_id = len(model.layers)-2
        self.model = Model(inputs=model.layers[0].input, outputs=model.layers[layer_id].output)
        self.softmax = Sequential()
        self.softmax.add(Lambda(lambda X: softmax(X, axis=1), input_shape=(10,))) 
Example 29
Project: EvadeML-Zoo   Author: mzweilin   File: magnet_mnist.py    MIT License 5 votes vote down vote up
def classify(self, X, option="logit", T=1):
        if option == "logit":
            return self.model.predict(X)
        if option == "prob":
            logits = self.model.predict(X)/T
            return self.softmax.predict(logits) 
Example 30
Project: ECG_Heartbeat_Classification   Author: CVxTz   File: baseline_mitbih.py    MIT License 5 votes vote down vote up
def get_model():
    nclass = 5
    inp = Input(shape=(187, 1))
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(inp)
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = Dropout(rate=0.1)(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = GlobalMaxPool1D()(img_1)
    img_1 = Dropout(rate=0.2)(img_1)

    dense_1 = Dense(64, activation=activations.relu, name="dense_1")(img_1)
    dense_1 = Dense(64, activation=activations.relu, name="dense_2")(dense_1)
    dense_1 = Dense(nclass, activation=activations.softmax, name="dense_3_mitbih")(dense_1)

    model = models.Model(inputs=inp, outputs=dense_1)
    opt = optimizers.Adam(0.001)

    model.compile(optimizer=opt, loss=losses.sparse_categorical_crossentropy, metrics=['acc'])
    model.summary()
    return model 
Example 31
Project: EEG_classification   Author: CVxTz   File: models.py    Apache License 2.0 5 votes vote down vote up
def get_model():
    nclass = 5
    inp = Input(shape=(3000, 1))
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(inp)
    img_1 = Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = SpatialDropout1D(rate=0.01)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = SpatialDropout1D(rate=0.01)(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = MaxPool1D(pool_size=2)(img_1)
    img_1 = SpatialDropout1D(rate=0.01)(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")(img_1)
    img_1 = GlobalMaxPool1D()(img_1)
    img_1 = Dropout(rate=0.01)(img_1)

    dense_1 = Dropout(rate=0.01)(Dense(64, activation=activations.relu, name="dense_1")(img_1))
    dense_1 = Dropout(rate=0.05)(Dense(64, activation=activations.relu, name="dense_2")(dense_1))
    dense_1 = Dense(nclass, activation=activations.softmax, name="dense_3")(dense_1)

    model = models.Model(inputs=inp, outputs=dense_1)
    opt = optimizers.Adam(0.001)

    model.compile(optimizer=opt, loss=losses.sparse_categorical_crossentropy, metrics=['acc'])
    model.summary()
    return model 
Example 32
Project: EEG_classification   Author: CVxTz   File: models.py    Apache License 2.0 5 votes vote down vote up
def get_model_cnn():
    nclass = 5

    seq_input = Input(shape=(None, 3000, 1))
    base_model = get_base_model()
    # for layer in base_model.layers:
    #     layer.trainable = False
    encoded_sequence = TimeDistributed(base_model)(seq_input)
    encoded_sequence = SpatialDropout1D(rate=0.01)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="relu",
                                                               padding="same")(encoded_sequence))
    encoded_sequence = Dropout(rate=0.05)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="relu",
                                                               padding="same")(encoded_sequence))

    #out = TimeDistributed(Dense(nclass, activation="softmax"))(encoded_sequence)
    out = Convolution1D(nclass, kernel_size=3, activation="softmax", padding="same")(encoded_sequence)

    model = models.Model(seq_input, out)

    model.compile(optimizers.Adam(0.001), losses.sparse_categorical_crossentropy, metrics=['acc'])
    model.summary()

    return model 
Example 33
Project: EEG_classification   Author: CVxTz   File: models.py    Apache License 2.0 5 votes vote down vote up
def get_model_cnn_crf(lr=0.001):
    nclass = 5

    seq_input = Input(shape=(None, 3000, 1))
    base_model = get_base_model()
    # for layer in base_model.layers:
    #     layer.trainable = False
    encoded_sequence = TimeDistributed(base_model)(seq_input)
    encoded_sequence = SpatialDropout1D(rate=0.01)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="relu",
                                                               padding="same")(encoded_sequence))
    encoded_sequence = Dropout(rate=0.05)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="linear",
                                                               padding="same")(encoded_sequence))

    #out = TimeDistributed(Dense(nclass, activation="softmax"))(encoded_sequence)
    # out = Convolution1D(nclass, kernel_size=3, activation="linear", padding="same")(encoded_sequence)

    crf = CRF(nclass, sparse_target=True)

    out = crf(encoded_sequence)


    model = models.Model(seq_input, out)

    model.compile(optimizers.Adam(lr), crf.loss_function, metrics=[crf.accuracy])
    model.summary()

    return model 
Example 34
Project: Capsule-Network   Author: zhanpenghe   File: keras_capsnet.py    MIT License 5 votes vote down vote up
def _dynamic_routing(self, u_hat, b_ij):
        
        for i in range(self.iterations):

            c_ij = softmax(b_ij, axis=1)
            s_j = K.batch_dot(c_ij, u_hat, [2, 2])
            v_j = squash(s_j)

            if i < self.iterations-1:
                b_ij +=  K.batch_dot(v_j, u_hat, [2, 3])
                # b_ij = b_ij + K.batch_dot(K.tile(v_j,  [1, self.prev_shape[1], 1, 1, 1]), u_hat, [3, 4])

        # return K.squeeze(v_j, axis=1)
        return v_j 
Example 35
Project: DeepCell-Keras   Author: Shenggan   File: helper.py    MIT License 5 votes vote down vote up
def axis_softmax(x, axis=1):
	return activations.softmax(x, axis=axis) 
Example 36
Project: DeepCell-Keras   Author: Shenggan   File: helper.py    MIT License 5 votes vote down vote up
def categorical_crossentropy(target, output, class_weights=None, axis=None, from_logits=False):
	"""Categorical crossentropy between an output tensor and a target tensor.
	# Arguments
		target: A tensor of the same shape as `output`.
		output: A tensor resulting from a softmax
		(unless `from_logits` is True, in which
		case `output` is expected to be the logits).
		from_logits: Boolean, whether `output` is the
		result of a softmax, or is a tensor of logits.
	# Returns
		Output tensor.
	"""
	# Note: tf.nn.softmax_cross_entropy_with_logits
	# expects logits, Keras expects probabilities.
	if axis is None:
		axis = len(output.get_shape()) - 1
	if not from_logits:
		# scale preds so that the class probas of each sample sum to 1
		output /= tf.reduce_sum(output, axis=axis, keep_dims=True)
		# manual computation of crossentropy
		_epsilon = _to_tensor(K.epsilon(), output.dtype.base_dtype)
		output = tf.clip_by_value(output, _epsilon, 1. - _epsilon)
		if class_weights is None:
			return -tf.reduce_sum(target * tf.log(output), axis=axis)
		return -tf.reduce_sum(tf.multiply(target * tf.log(output), class_weights), axis=axis)

	return tf.nn.softmax_cross_entropy_with_logits(labels=target, logits=output) 
Example 37
Project: CIKM-AnalytiCup-2018   Author: zake7749   File: utils.py    Apache License 2.0 5 votes vote down vote up
def __call__(self, q, k, v, mask):
        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])
        if mask is not None:
            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)
            attn = Add()([attn, mmask])
        attn = Activation('softmax')(attn)
        attn = self.dropout(attn)
        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])
        return output, attn 
Example 38
Project: CIKM-AnalytiCup-2018   Author: zake7749   File: utils.py    Apache License 2.0 5 votes vote down vote up
def soft_attention_alignment(input_1, input_2):
    "Align text representation with neural soft attention"
    attention = Dot(axes=-1)([input_1, input_2])
    w_att_1 = Lambda(lambda x: softmax(x, axis=1),
                     output_shape=unchanged_shape)(attention)
    w_att_2 = Permute((2, 1))(Lambda(lambda x: softmax(x, axis=2),
                             output_shape=unchanged_shape)(attention))
    in1_aligned = Dot(axes=1)([w_att_1, input_1])
    in2_aligned = Dot(axes=1)([w_att_2, input_2])
    return in1_aligned, in2_aligned 
Example 39
Project: C3AE   Author: StevenBanama   File: C3AE.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def KeepTopN(input, keep=2):
    values, indices = tf.nn.top_k(input, k=keep, sorted=False)
    thres = tf.reduce_min(values)
    mask = tf.greater_equal(input, thres)
    return tf.nn.softmax(tf.where(mask, input, tf.zeros_like(input))) 
Example 40
Project: C3AE   Author: StevenBanama   File: C3AE.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def KeepMaxAdjactent(input, keep=2, axis=-1):
    dims = int(input.get_shape()[axis])
    combine_val = tf.gather(input, [x for x in xrange(1, dims)], axis=axis) + tf.gather(input, [x for x in xrange(0, dims-1)], axis=axis)
    indices = tf.argmax(combine_val, axis=axis)
    window_ind = tf.stack([indices, indices+1], axis=-1)
    print(window_ind)
    val = tf.gather_nd(input, window_ind)
    print(val)
    # 下面的tensor记得要改改, https://stackoverflow.com/questions/37001686/using-sparsetensor-as-a-trainable-variable
    # validate_indices 一定要设置成False否则过不去
    window_mat = tf.sparse_tensor_to_dense(tf.SparseTensor(window_ind, val, dense_shape=tf.shape(input, out_type=tf.int64)), validate_indices=False)
    print(window_mat)
    return tf.nn.softmax(window_mat) 
Example 41
Project: DeepPavlov   Author: deepmipt   File: keras_layers.py    Apache License 2.0 5 votes vote down vote up
def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):
    """
    Compute additive self attention for time series of vectors (with batch dimension)
            the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
            v is a learnable vector of n_hidden dimensionality,
            W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of2784131 units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
        """
    n_input_features = K.int_shape(units)[2]
    if n_hidden is None:
        n_hidden = n_input_features
    if n_output_features is None:
        n_output_features = n_input_features
    exp1 = Lambda(lambda x: expand_tile(x, axis=1))(units)
    exp2 = Lambda(lambda x: expand_tile(x, axis=2))(units)
    units_pairs = Concatenate(axis=3)([exp1, exp2])
    query = Dense(n_hidden, activation="tanh")(units_pairs)
    attention = Dense(1, activation=lambda x: softmax(x, axis=2))(query)
    attended_units = Lambda(lambda x: K.sum(attention * x, axis=2))(exp1)
    output = Dense(n_output_features, activation=activation)(attended_units)
    return output 
Example 42
Project: DeepPavlov   Author: deepmipt   File: keras_layers.py    Apache License 2.0 5 votes vote down vote up
def multiplicative_self_attention(units, n_hidden=None, n_output_features=None, activation=None):
    """
    Compute multiplicative self attention for time series of vectors (with batch dimension)
    the formula: score(h_i, h_j) = <W_1 h_i,  W_2 h_j>,  W_1 and W_2 are learnable matrices
    with dimensionality [n_hidden, n_input_features]

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
    """
    n_input_features = K.int_shape(units)[2]
    if n_hidden is None:
        n_hidden = n_input_features
    if n_output_features is None:
        n_output_features = n_input_features
    exp1 = Lambda(lambda x: expand_tile(x, axis=1))(units)
    exp2 = Lambda(lambda x: expand_tile(x, axis=2))(units)
    queries = Dense(n_hidden)(exp1)
    keys = Dense(n_hidden)(exp2)
    scores = Lambda(lambda x: K.sum(queries * x, axis=3, keepdims=True))(keys)
    attention = Lambda(lambda x: softmax(x, axis=2))(scores)
    mult = Multiply()([attention, exp1])
    attended_units = Lambda(lambda x: K.sum(x, axis=2))(mult)
    output = Dense(n_output_features, activation=activation)(attended_units)
    return output 
Example 43
Project: algorimp   Author: marczellm   File: _main.py    GNU General Public License v3.0 5 votes vote down vote up
def _build_net(self) -> keras.models.Model:
        input_tensor = keras.layers.Input(shape=self.inputshape())
        hidden_tensor = keras.layers.Dense(800, activation=relu)(input_tensor)

        pitch_tensor = keras.layers.Dense(127, activation=softmax)(hidden_tensor)
        tsbq_tensor = keras.layers.Dense(self.maxtsbq + 1, activation=softmax)(hidden_tensor)
        dq_tensor = keras.layers.Dense(self.maxdq + 1, activation=softmax)(hidden_tensor)

        model = keras.models.Model(inputs=input_tensor, outputs=[pitch_tensor, tsbq_tensor, dq_tensor])
        model.compile(optimizer=adagrad(), loss=categorical_crossentropy)

        self.epochs = 20
        self.outfuns = sampler(.3), weighted_nlargest(2), weighted_nlargest(2)
        return model 
Example 44
Project: algorimp   Author: marczellm   File: _main.py    GNU General Public License v3.0 5 votes vote down vote up
def _build_net(self) -> keras.models.Model:
        input_tensor = keras.layers.Input(shape=self.inputshape())
        hidden_tensor = keras.layers.Dense(800, activation=relu)(input_tensor)
        hidden_tensor = keras.layers.Dense(800, activation=relu)(hidden_tensor)

        pitch_tensor = keras.layers.Dense(127, activation=softmax)(hidden_tensor)
        tsbq_tensor = keras.layers.Dense(self.maxtsbq + 1, activation=softmax)(hidden_tensor)
        dq_tensor = keras.layers.Dense(self.maxdq + 1, activation=softmax)(hidden_tensor)

        model = keras.models.Model(inputs=input_tensor, outputs=[pitch_tensor, tsbq_tensor, dq_tensor])
        model.compile(optimizer=adagrad(), loss=categorical_crossentropy)

        self.epochs = 25
        self.outfuns = (sampler(.5),) * 3
        return model 
Example 45
Project: algorimp   Author: marczellm   File: _main.py    GNU General Public License v3.0 5 votes vote down vote up
def _build_net(self) -> keras.models.Model:
        dummy_input = self._encode_network_input([Note()] * self.order,
                                                 [Chord('C7')] * self.chord_order, self.changes)
        in_notes = keras.layers.Input(batch_shape=(1,) + dummy_input[0].shape) if self.stateful\
            else keras.layers.Input(shape=dummy_input[0].shape)
        in_chords = keras.layers.Input(batch_shape=(1,) + dummy_input[1].shape) if self.stateful\
            else keras.layers.Input(shape=dummy_input[1].shape)

        lstm_out = keras.layers.LSTM(512, stateful=self.stateful, implementation=self._implementation)(in_notes)
        x = keras.layers.concatenate([lstm_out, in_chords])
        x = keras.layers.Dense(512)(x)

        pitch_tensor = keras.layers.Dense(12, activation=softmax)(x)
        tsbq_tensor = keras.layers.Dense(self.maxtsbq + 1, activation=softmax)(x)
        dq_tensor = keras.layers.Dense(self.maxdq + 1, activation=softmax)(x)
        octave_tensor = keras.layers.Dense(NUM_OCTAVES, activation=softmax)(x)
        beatdiff_tensor = keras.layers.Dense(self.maxbeatdiff + 1, activation=softmax)(x)

        model = keras.models.Model(inputs=[in_notes, in_chords],
                                   outputs=[pitch_tensor, tsbq_tensor, dq_tensor, octave_tensor, beatdiff_tensor])
        model.compile(optimizer=rmsprop(), loss=categorical_crossentropy)

        self.octave_model = keras.models.Model(inputs=model.inputs, outputs=model.outputs[3])
        self.epochs = 30
        self.outfuns = (sampler(.1),) + (weighted_nlargest(2),) * 2 + (np.argmax,) * 2

        return model 
Example 46
Project: nlp_research   Author: zhufz   File: esim.py    MIT License 4 votes vote down vote up
def __call__(self, x_query, x_sample, reuse = tf.AUTO_REUSE, **kwargs):
        #embedding_sequence_q1 = BatchNormalization(axis=2)(x_query)
        #embedding_sequence_q2 = BatchNormalization(axis=2)(x_sample)
        #final_embedding_sequence_q1 = SpatialDropout1D(0.25)(embedding_sequence_q1)
        #final_embedding_sequence_q2 = SpatialDropout1D(0.25)(embedding_sequence_q2)

        #################### 输入编码input encoding #######################
        #分别对query和sample进行双向编码
        rnn_layer_q1 = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(x_query)
        rnn_layer_q2 = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(x_sample)
        #rnn_layer_q1 = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(final_embedding_sequence_q1)
        #rnn_layer_q2 = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(final_embedding_sequence_q2)

        ############## 局部推理local inference modeling ###################
        #计算dot attention
        attention = Dot(axes=-1)([rnn_layer_q1, rnn_layer_q2])
        #分别计算query和sample进行attention后的结果
        w_attn_1 = Lambda(lambda x: softmax(x, axis=1))(attention)
        w_attn_2 = Permute((2, 1))(Lambda(lambda x: softmax(x, axis=2))(attention))
        align_layer_1 = Dot(axes=1)([w_attn_1, rnn_layer_q1])
        align_layer_2 = Dot(axes=1)([w_attn_2, rnn_layer_q2])

        ############# 推理组合Inference Composition #######################

        subtract_layer_1 = subtract([rnn_layer_q1, align_layer_1])
        subtract_layer_2 = subtract([rnn_layer_q2, align_layer_2])

        multiply_layer_1 = multiply([rnn_layer_q1, align_layer_1])
        multiply_layer_2 = multiply([rnn_layer_q2, align_layer_2])

        m_q1 = concatenate([rnn_layer_q1, align_layer_1, subtract_layer_1, multiply_layer_1])
        m_q2 = concatenate([rnn_layer_q2, align_layer_2, subtract_layer_2, multiply_layer_2])

        ############### 编码+池化 #######################
        v_q1_i = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(m_q1)
        v_q2_i = Bidirectional(LSTM(self.recurrent_units, return_sequences=True))(m_q2)

        avgpool_q1 = GlobalAveragePooling1D()(v_q1_i)
        avgpool_q2 = GlobalAveragePooling1D()(v_q2_i)
        maxpool_q1 = GlobalMaxPooling1D()(v_q1_i)
        maxpool_q2 = GlobalMaxPooling1D()(v_q2_i)

        merged_q1 = concatenate([avgpool_q1, maxpool_q1])
        merged_q2 = concatenate([avgpool_q2, maxpool_q2])

        final_v = BatchNormalization()(concatenate([merged_q1, merged_q2]))
        #output = Dense(units=self.dense_units, activation='relu')(final_v)
        output = Dense(units=self.num_output, activation=None)(final_v)
        #output = BatchNormalization()(output)
        #output = Dropout(self.dropout_rate)(output)
        #output = tf.nn.dropout(output, self.keep_prob)
        #高级api tf.layer.dropout 与 keras的Dropout都使用dropout
        #tf.nn.dropout使用keep_prob
        #output = Dense(units=self.num_output, activation='sigmoid')(output)
        #output = Dense(units=self.num_output, activation=None)(output)
        #output = tf.squeeze(output, -1)
        return output 
Example 47
Project: ntm_keras   Author: flomlo   File: ntm.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def _split_and_apply_activations(self, controller_output):
        """ This takes the controller output, splits it in ntm_output, read and wright adressing data.
            It returns a triple of ntm_output, controller_instructions_read, controller_instructions_write.
            ntm_output is a tensor, controller_instructions_read and controller_instructions_write are lists containing
            the adressing instruction (k, beta, g, shift, gamma) and in case of write also the writing constructions,
            consisting of an erase and an add vector. 

            As it is necesseary for stable results,
            k and add_vector is activated via tanh, erase_vector via sigmoid (this is critical!),
            shift via softmax,
            gamma is sigmoided, inversed and clipped (probably not ideal)
            g is sigmoided,
            beta is linear (probably not ideal!) """
        
        # splitting
        ntm_output, controller_instructions_read, controller_instructions_write = tf.split(
                    controller_output,
                    np.asarray([self.output_dim,
                                self.read_heads * self.controller_read_head_emitting_dim,
                                self.write_heads * self.controller_write_head_emitting_dim]),
                    axis=1)

        controller_instructions_read = tf.split(controller_instructions_read, self.read_heads, axis=1)
        controller_instructions_write = tf.split(controller_instructions_write, self.write_heads, axis=1)

        controller_instructions_read = [
                tf.split(single_head_data, np.asarray([self.m_depth, 1, 1, 3, 1]), axis=1) for 
                single_head_data in controller_instructions_read]
        
        controller_instructions_write = [
                tf.split(single_head_data, np.asarray([self.m_depth, 1, 1, 3, 1, self.m_depth, self.m_depth]), axis=1) for 
                single_head_data in controller_instructions_write]
        
        #activation
        ntm_output = self.activation(ntm_output)
        controller_instructions_read = [(tanh(k), hard_sigmoid(beta)+0.5, sigmoid(g), softmax(shift), 1 + 9*sigmoid(gamma)) for
                (k, beta, g, shift, gamma) in controller_instructions_read]
        controller_instructions_write = [
                (tanh(k), hard_sigmoid(beta)+0.5, sigmoid(g), softmax(shift), 1 + 9*sigmoid(gamma), hard_sigmoid(erase_vector), tanh(add_vector))  for 
                (k, beta, g, shift, gamma, erase_vector, add_vector) in controller_instructions_write]
       
        return (ntm_output, controller_instructions_read, controller_instructions_write) 
Example 48
Project: CNNArt   Author: thomaskuestner   File: MNetArt.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                 dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    l2_reg = 1e-4

    # (4 stages-each 2 convs)(378,722 params)(for 40x40x10)
    input_t = Input(shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2])))
    input2D_t = Permute((4, 1, 2, 3))(input_t)
    input2D_t = Reshape(target_shape=(int(patchSize[0, 2]), int(patchSize[0, 0]), int(patchSize[0, 1])))(
        input2D_t)
    # use zDimension as number of channels
    twoD_t = Conv2D(16,
                    kernel_size=(7, 7),
                    padding='same',
                    kernel_initializer='he_normal',
                    kernel_regularizer=l2(l2_reg),
                    strides=(1, 1)
                    )(input2D_t)
    twoD_t = Activation('relu')(twoD_t)

    l_w2_t = fCreateMaxPooling2D(twoD_t, stride=(2, 2))
    l_w3_t = fCreateMaxPooling2D(l_w2_t, stride=(2, 2))
    l_w4_t = fCreateMaxPooling2D(l_w3_t, stride=(2, 2))

    stage1_res1_t = fCreateMNet_Block(twoD_t, 16, kernel_size=(3, 3), forwarding=True, l2_reg=l2_reg)
    stage1_res2_t = fCreateMNet_Block(stage1_res1_t, 32, kernel_size=(3, 3), forwarding=False, l2_reg=l2_reg)

    stage2_inp_t = fCreateMaxPooling2D(stage1_res2_t, stride=(2, 2))
    stage2_inp_t = concatenate([stage2_inp_t, l_w2_t], axis=1)
    stage2_res1_t = fCreateMNet_Block(stage2_inp_t, 32, l2_reg=l2_reg)
    stage2_res2_t = fCreateMNet_Block(stage2_res1_t, 48, forwarding=False)

    stage3_inp_t = fCreateMaxPooling2D(stage2_res2_t, stride=(2, 2))
    stage3_inp_t = concatenate([stage3_inp_t, l_w3_t], axis=1)
    stage3_res1_t = fCreateMNet_Block(stage3_inp_t, 48, l2_reg=l2_reg)
    stage3_res2_t = fCreateMNet_Block(stage3_res1_t, 64, forwarding=False, l2_reg=l2_reg)

    stage4_inp_t = fCreateMaxPooling2D(stage3_res2_t, stride=(2, 2))
    stage4_inp_t = concatenate([stage4_inp_t, l_w4_t], axis=1)
    stage4_res1_t = fCreateMNet_Block(stage4_inp_t, 64, l2_reg=l2_reg)
    stage4_res2_t = fCreateMNet_Block(stage4_res1_t, 128, forwarding=False, l2_reg=l2_reg)

    after_flat_t = Flatten()(stage4_res2_t)

    after_dense_t = Dense(units=2,
                          kernel_initializer='he_normal',
                          kernel_regularizer=l2(l2_reg))(after_flat_t)
    output_t = Activation('softmax')(after_dense_t)

    cnn = Model(inputs=[input_t], outputs=[output_t])

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)
    cnn.compile(optimizer=opti, loss=loss, metrics=['accuracy'])
    sArchiSpecs = '3stages_l2{}'.format(l2_reg) 
Example 49
Project: CNNArt   Author: thomaskuestner   File: CNN3D.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                 dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    # change to functional API
    input_t = Input(shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2])))
    seq_t = Dropout(dr_rate)(input_t)
    seq_t = Conv3D(32,  # numChans
                   kernel_size=(14, 14, 5),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg),
                   input_shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2]))
                   )(seq_t)
    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Conv3D(64,
                   kernel_size=(7, 7, 3),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg))(seq_t)

    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Conv3D(128,
                   kernel_size=(3, 3, 2),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg))(seq_t)

    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Flatten()(seq_t)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Dense(units=2,
                  kernel_initializer='normal',
                  kernel_regularizer=l2(l2_reg))(seq_t)
    output_t = Activation('softmax')(seq_t)

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)  # loss cat_crosent default

    cnn = Model(inputs=[input_t], outputs=[output_t])
    cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
    sArchiSpecs = '_l2{}'.format(l2_reg)


####################################################################helpers############################################# 
Example 50
Project: CNNArt   Author: thomaskuestner   File: MNetArt.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                 dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    l2_reg = 1e-4

    # (4 stages-each 2 convs)(378,722 params)(for 40x40x10)
    input_t = Input(shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2])))
    input2D_t = Permute((4, 1, 2, 3))(input_t)
    input2D_t = Reshape(target_shape=(int(patchSize[0, 2]), int(patchSize[0, 0]), int(patchSize[0, 1])))(
        input2D_t)
    # use zDimension as number of channels
    twoD_t = Conv2D(16,
                    kernel_size=(7, 7),
                    padding='same',
                    kernel_initializer='he_normal',
                    kernel_regularizer=l2(l2_reg),
                    strides=(1, 1)
                    )(input2D_t)
    twoD_t = Activation('relu')(twoD_t)

    l_w2_t = fCreateMaxPooling2D(twoD_t, stride=(2, 2))
    l_w3_t = fCreateMaxPooling2D(l_w2_t, stride=(2, 2))
    l_w4_t = fCreateMaxPooling2D(l_w3_t, stride=(2, 2))

    stage1_res1_t = fCreateMNet_Block(twoD_t, 16, kernel_size=(3, 3), forwarding=True, l2_reg=l2_reg)
    stage1_res2_t = fCreateMNet_Block(stage1_res1_t, 32, kernel_size=(3, 3), forwarding=False, l2_reg=l2_reg)

    stage2_inp_t = fCreateMaxPooling2D(stage1_res2_t, stride=(2, 2))
    stage2_inp_t = concatenate([stage2_inp_t, l_w2_t], axis=1)
    stage2_res1_t = fCreateMNet_Block(stage2_inp_t, 32, l2_reg=l2_reg)
    stage2_res2_t = fCreateMNet_Block(stage2_res1_t, 48, forwarding=False)

    stage3_inp_t = fCreateMaxPooling2D(stage2_res2_t, stride=(2, 2))
    stage3_inp_t = concatenate([stage3_inp_t, l_w3_t], axis=1)
    stage3_res1_t = fCreateMNet_Block(stage3_inp_t, 48, l2_reg=l2_reg)
    stage3_res2_t = fCreateMNet_Block(stage3_res1_t, 64, forwarding=False, l2_reg=l2_reg)

    stage4_inp_t = fCreateMaxPooling2D(stage3_res2_t, stride=(2, 2))
    stage4_inp_t = concatenate([stage4_inp_t, l_w4_t], axis=1)
    stage4_res1_t = fCreateMNet_Block(stage4_inp_t, 64, l2_reg=l2_reg)
    stage4_res2_t = fCreateMNet_Block(stage4_res1_t, 128, forwarding=False, l2_reg=l2_reg)

    after_flat_t = Flatten()(stage4_res2_t)

    after_dense_t = Dense(units=2,
                          kernel_initializer='he_normal',
                          kernel_regularizer=l2(l2_reg))(after_flat_t)
    output_t = Activation('softmax')(after_dense_t)

    cnn = Model(inputs=[input_t], outputs=[output_t])

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)
    cnn.compile(optimizer=opti, loss=loss, metrics=['accuracy'])
    sArchiSpecs = '3stages_l2{}'.format(l2_reg) 
Example 51
Project: CNNArt   Author: thomaskuestner   File: 3D_CNN.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                 dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
    # change to functional API
    input_t = Input(shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2])))
    seq_t = Dropout(dr_rate)(input_t)
    seq_t = Conv3D(32,  # numChans
                   kernel_size=(14, 14, 5),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg),
                   input_shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]), int(patchSize[0, 2]))
                   )(seq_t)
    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Conv3D(64,
                   kernel_size=(7, 7, 3),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg))(seq_t)

    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Conv3D(128,
                   kernel_size=(3, 3, 2),
                   kernel_initializer='he_normal',
                   weights=None,
                   padding='valid',
                   strides=(1, 1, 1),
                   kernel_regularizer=l2(l2_reg))(seq_t)

    seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

    seq_t = Flatten()(seq_t)

    seq_t = Dropout(dr_rate)(seq_t)
    seq_t = Dense(units=2,
                  kernel_initializer='normal',
                  kernel_regularizer=l2(l2_reg))(seq_t)
    output_t = Activation('softmax')(seq_t)

    opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)  # loss cat_crosent default

    cnn = Model(inputs=[input_t], outputs=[output_t])
    cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
    sArchiSpecs = '_l2{}'.format(l2_reg)


####################################################################helpers############################################# 
Example 52
Project: CNNArt   Author: thomaskuestner   File: motion_MNetArt.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
        dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
        l2_reg=1e-4

        #(4 stages-each 2 convs)(378,722 params)(for 40x40x10)
        input_t=Input(shape=(1,int(patchSize[0, 0]),int(patchSize[0, 1]), int(patchSize[0, 2])))
        input2D_t=Permute((4,1,2,3))(input_t)
        input2D_t=Reshape(target_shape=(int(patchSize[0, 2]),int(patchSize[0, 0]), int(patchSize[0, 1])))(
            input2D_t)
        #use zDimension as number of channels
        twoD_t=Conv2D(16,
                      kernel_size=(7,7),
                      padding='same',
                      kernel_initializer='he_normal',
                      kernel_regularizer=l2(l2_reg),
                      strides=(1,1)
                      )(input2D_t)
        twoD_t = Activation('relu')(twoD_t)

        l_w2_t = fCreateMaxPooling2D(twoD_t, stride=(2, 2))
        l_w3_t = fCreateMaxPooling2D(l_w2_t, stride=(2, 2))
        l_w4_t = fCreateMaxPooling2D(l_w3_t, stride=(2, 2))

        stage1_res1_t=fCreateMNet_Block(twoD_t,16,kernel_size=(3,3), forwarding=True, l2_reg=l2_reg)
        stage1_res2_t=fCreateMNet_Block(stage1_res1_t,32,kernel_size=(3,3), forwarding=False, l2_reg=l2_reg)

        stage2_inp_t=fCreateMaxPooling2D(stage1_res2_t, stride=(2,2))
        stage2_inp_t=concatenate([stage2_inp_t,l_w2_t], axis=1)
        stage2_res1_t=fCreateMNet_Block(stage2_inp_t,32,l2_reg=l2_reg)
        stage2_res2_t=fCreateMNet_Block(stage2_res1_t,48, forwarding=False)

        stage3_inp_t=fCreateMaxPooling2D(stage2_res2_t, stride=(2,2))
        stage3_inp_t=concatenate([stage3_inp_t,l_w3_t], axis=1)
        stage3_res1_t=fCreateMNet_Block(stage3_inp_t,48,l2_reg=l2_reg)
        stage3_res2_t = fCreateMNet_Block(stage3_res1_t, 64, forwarding=False,l2_reg=l2_reg)

        stage4_inp_t = fCreateMaxPooling2D(stage3_res2_t, stride=(2, 2))
        stage4_inp_t = concatenate([stage4_inp_t, l_w4_t], axis=1)
        stage4_res1_t = fCreateMNet_Block(stage4_inp_t, 64,l2_reg=l2_reg)
        stage4_res2_t = fCreateMNet_Block(stage4_res1_t, 128, forwarding=False,l2_reg=l2_reg)

        after_flat_t = Flatten()(stage4_res2_t)

        after_dense_t = Dense(units=2,
                              kernel_initializer='he_normal',
                              kernel_regularizer=l2(l2_reg))(after_flat_t)
        output_t = Activation('softmax')(after_dense_t)

        cnn = Model(inputs=[input_t], outputs=[output_t])

        opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)
        cnn.compile(optimizer=opti, loss=loss, metrics=['accuracy'])
        sArchiSpecs = '3stages_l2{}'.format(l2_reg) 
Example 53
Project: CNNArt   Author: thomaskuestner   File: motion_CNN3D.py    Apache License 2.0 4 votes vote down vote up
def fCreateModel(patchSize, learningRate=1e-3, optimizer='SGD',
                     dr_rate=0.0, input_dr_rate=0.0, max_norm=5, iPReLU=0, l2_reg=1e-6):
# change to functional API
        input_t = Input(shape=(1, int(patchSize[0]), int(patchSize[1]), int(patchSize[2])))
        seq_t= Dropout(dr_rate)(input_t)
        seq_t = Conv3D(32,  # numChans
                       kernel_size=(14, 14, 5),
                       kernel_initializer='he_normal',
                       weights=None,
                       padding='valid',
                       strides=(1, 1, 1),
                       kernel_regularizer=l2(l2_reg),
                       input_shape=(1, int(patchSize[0]), int(patchSize[1]), int(patchSize[2]))
                       )(seq_t)
        seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

        seq_t = Dropout(dr_rate)(seq_t)
        seq_t = Conv3D(64,
                       kernel_size=(7, 7, 3),
                       kernel_initializer='he_normal',
                       weights=None,
                       padding='valid',
                       strides=(1, 1, 1),
                       kernel_regularizer=l2(l2_reg))(seq_t)

        seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

        seq_t = Dropout(dr_rate)(seq_t)
        seq_t = Conv3D(128,
                       kernel_size=(3, 3, 2),
                       kernel_initializer='he_normal',
                       weights=None,
                       padding='valid',
                       strides=(1, 1, 1),
                       kernel_regularizer=l2(l2_reg))(seq_t)

        seq_t = fGetActivation(seq_t, iPReLU=iPReLU)

        seq_t = Flatten()(seq_t)

        seq_t = Dropout(dr_rate)(seq_t)
        seq_t = Dense(units=2,
                      kernel_initializer='normal',
                      kernel_regularizer=l2(l2_reg))(seq_t)
        output_t = Activation('softmax')(seq_t)

        opti, loss = fGetOptimizerAndLoss(optimizer, learningRate=learningRate)  # loss cat_crosent default

        cnn = Model(inputs=[input_t], outputs=[output_t])
        cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
        sArchiSpecs = '_l2{}'.format(l2_reg)

        return cnn


####################################################################helpers############################################# 
Example 54
Project: NPRF   Author: ucasir   File: nprf_drmm.py    Apache License 2.0 4 votes vote down vote up
def build(self):

    dd_q_input = Input((self.config.nb_supervised_doc, self.config.doc_topk_term, 1), name='dd_q_input')
    dd_d_input = Input((self.config.nb_supervised_doc, self.config.doc_topk_term,
                        self.config.hist_size), name='dd_d_input')

    dd_q_w = Dense(1, kernel_initializer=self.initializer_gate, use_bias=False, name='dd_q_gate')(dd_q_input)
    dd_q_w = Lambda(lambda x: softmax(x, axis=2), output_shape=(
                  self.config.nb_supervised_doc, self.config.doc_topk_term,), name='dd_q_softmax')(dd_q_w)

    z = dd_d_input
    for i in range(self.config.nb_layers):
      z = Dense(self.config.hidden_size[i], activation='tanh',
                kernel_initializer=self.initializer_fc, name='hidden')(z)
    z = Dense(self.config.out_size, kernel_initializer=self.initializer_fc, name='dd_d_gate')(z)
    z = Reshape((self.config.nb_supervised_doc, self.config.doc_topk_term,))(z)
    dd_q_w = Reshape((self.config.nb_supervised_doc, self.config.doc_topk_term,))(dd_q_w)
    # out = Dot(axes=[2, 2], name='dd_pseudo_out')([z, dd_q_w])

    out = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[2, 2]), name='dd_pseudo_out')([z, dd_q_w])
    dd_init_out = Lambda(lambda x: tf.matrix_diag_part(x), output_shape=(self.config.nb_supervised_doc,), name='dd_init_out')(out)
    '''
    dd_init_out = Lambda(lambda x: tf.reduce_sum(x, axis=2), output_shape=(self.config.nb_supervised_doc,))(z)
    '''
    #dd_out = Reshape((self.config.nb_supervised_doc,))(dd_out)

    # dd out gating
    dd_gate = Input((self.config.nb_supervised_doc, 1), name='baseline_doc_score')
    dd_w = Dense(1, kernel_initializer=self.initializer_gate, use_bias=False, name='dd_gate')(dd_gate)
    # dd_w = Lambda(lambda x: softmax(x, axis=1), output_shape=(self.config.nb_supervised_doc,), name='dd_softmax')(dd_w)

    # dd_out = Dot(axes=[1, 1], name='dd_out')([dd_init_out, dd_w])
    dd_w = Reshape((self.config.nb_supervised_doc,))(dd_w)
    dd_init_out = Reshape((self.config.nb_supervised_doc,))(dd_init_out)


    if self.config.method in [1, 3]: # no doc gating, with dense layer
      z = dd_init_out
    elif self.config.method == 2:
      logging.info("Apply doc gating")
      z = Multiply(name='dd_out')([dd_init_out, dd_w])
    else:
      raise ValueError("Method not initialized, please check config file")

    if self.config.method in [1, 2]:
      logging.info("Dense layer on top")
      z = Dense(self.config.merge_hidden, activation='tanh', name='merge_hidden')(z)
      out = Dense(self.config.merge_out, name='score')(z)
    else:
      logging.info("Apply doc gating, No dense layer on top, sum up scores")
      out = Dot(axes=[1, 1], name='score')([z, dd_w])

    model = Model(inputs=[dd_q_input, dd_d_input, dd_gate], outputs=[out])
    print(model.summary())

    return model 
Example 55
Project: keras-attention   Author: datalogue   File: custom_recurrents.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def step(self, x, states):

        ytm, stm = states

        # repeat the hidden state to the length of the sequence
        _stm = K.repeat(stm, self.timesteps)

        # now multiplty the weight matrix with the repeated hidden state
        _Wxstm = K.dot(_stm, self.W_a)

        # calculate the attention probabilities
        # this relates how much other timesteps contributed to this one.
        et = K.dot(activations.tanh(_Wxstm + self._uxpb),
                   K.expand_dims(self.V_a))
        at = K.exp(et)
        at_sum = K.sum(at, axis=1)
        at_sum_repeated = K.repeat(at_sum, self.timesteps)
        at /= at_sum_repeated  # vector of size (batchsize, timesteps, 1)

        # calculate the context vector
        context = K.squeeze(K.batch_dot(at, self.x_seq, axes=1), axis=1)
        # ~~~> calculate new hidden state
        # first calculate the "r" gate:

        rt = activations.sigmoid(
            K.dot(ytm, self.W_r)
            + K.dot(stm, self.U_r)
            + K.dot(context, self.C_r)
            + self.b_r)

        # now calculate the "z" gate
        zt = activations.sigmoid(
            K.dot(ytm, self.W_z)
            + K.dot(stm, self.U_z)
            + K.dot(context, self.C_z)
            + self.b_z)

        # calculate the proposal hidden state:
        s_tp = activations.tanh(
            K.dot(ytm, self.W_p)
            + K.dot((rt * stm), self.U_p)
            + K.dot(context, self.C_p)
            + self.b_p)

        # new hidden state:
        st = (1-zt)*stm + zt * s_tp

        yt = activations.softmax(
            K.dot(ytm, self.W_o)
            + K.dot(stm, self.U_o)
            + K.dot(context, self.C_o)
            + self.b_o)

        if self.return_probabilities:
            return at, [yt, st]
        else:
            return yt, [yt, st] 
Example 56
Project: TranskribusDU   Author: Transkribus   File: attentiondecoder.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def step(self, x, states):

        ytm, stm = states

        # repeat the hidden state to the length of the sequence
        _stm = K.repeat(stm, self.timesteps)

        # now multiplty the weight matrix with the repeated hidden state
        _Wxstm = K.dot(_stm, self.W_a)

        # calculate the attention probabilities
        # this relates how much other timesteps contributed to this one.
        et = K.dot(activations.tanh(_Wxstm + self._uxpb),
                   K.expand_dims(self.V_a))
        at = K.exp(et)
        at_sum = K.sum(at, axis=1)
        at_sum_repeated = K.repeat(at_sum, self.timesteps)
        at /= at_sum_repeated  # vector of size (batchsize, timesteps, 1)

        # calculate the context vector
        context = K.squeeze(K.batch_dot(at, self.x_seq, axes=1), axis=1)
        # ~~~> calculate new hidden state
        # first calculate the "r" gate:

        rt = activations.sigmoid(
            K.dot(ytm, self.W_r)
            + K.dot(stm, self.U_r)
            + K.dot(context, self.C_r)
            + self.b_r)

        # now calculate the "z" gate
        zt = activations.sigmoid(
            K.dot(ytm, self.W_z)
            + K.dot(stm, self.U_z)
            + K.dot(context, self.C_z)
            + self.b_z)

        # calculate the proposal hidden state:
        s_tp = activations.tanh(
            K.dot(ytm, self.W_p)
            + K.dot((rt * stm), self.U_p)
            + K.dot(context, self.C_p)
            + self.b_p)

        # new hidden state:
        st = (1-zt)*stm + zt * s_tp

        yt = activations.softmax(
            K.dot(ytm, self.W_o)
            + K.dot(stm, self.U_o)
            + K.dot(context, self.C_o)
            + self.b_o)

        if self.return_probabilities:
            return at, [yt, st]
        else:
            return yt, [yt, st]