Python keras_contrib.layers.CRF Examples

def train_model():
    if cxl_model:
        embedding_matrix = load_embedding()
    else:
        embedding_matrix = {}
    train, label = vocab_train_label(train_path, vocab=vocab, tags=tag, max_chunk_length=length)
    n = np.array(label, dtype=np.float)
    labels = n.reshape((n.shape[0], n.shape[1], 1))
    model = Sequential([
        Embedding(input_dim=len(vocab), output_dim=300, mask_zero=True, input_length=length, weights=[embedding_matrix],
                  trainable=False),
        SpatialDropout1D(0.2),
        Bidirectional(layer=LSTM(units=150, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)),
        TimeDistributed(Dense(len(tag), activation=relu)),
    ])
    crf_ = CRF(units=len(tag), sparse_target=True)
    model.add(crf_)
    model.compile(optimizer=Adam(), loss=crf_.loss_function, metrics=[crf_.accuracy])
    model.fit(x=np.array(train), y=labels, batch_size=16, epochs=4, callbacks=[RemoteMonitor()])
    model.save(model_path) 

def __build_model(self, emb_matrix=None):
        word_input = Input(shape=(None,), dtype='int32', name="word_input")

        word_emb = Embedding(self.vocab_size + 1, self.embed_dim,
                             weights=[emb_matrix] if emb_matrix is not None else None,
                             trainable=True if emb_matrix is None else False,
                             name='word_emb')(word_input)

        bilstm_output = Bidirectional(LSTM(self.bi_lstm_units // 2,
                                           return_sequences=True))(word_emb)

        bilstm_output = Dropout(self.dropout_rate)(bilstm_output)

        output = Dense(self.chunk_size + 1, kernel_initializer="he_normal")(bilstm_output)
        output = CRF(self.chunk_size + 1, sparse_target=self.sparse_target)(output)

        model = Model([word_input], [output])
        parallel_model = model
        if self.num_gpu > 1:
            parallel_model = multi_gpu_model(model, gpus=self.num_gpu)

        parallel_model.compile(optimizer=self.optimizer, loss=crf_loss, metrics=[crf_accuracy])
        return model, parallel_model 

def create_model(train=True):
    if train:
        (train_x, train_y), (test_x, test_y), (vocab, chunk_tags) = load_data()
    else:
        with open('model/config.pkl', 'rb') as inp:
            (vocab, chunk_tags) = pickle.load(inp)
    model = Sequential()
    model.add(Embedding(len(vocab), EMBED_DIM, mask_zero=True))  # Random embedding
    model.add(Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True)))
    crf = CRF(len(chunk_tags), sparse_target=True)
    model.add(crf)
    model.summary()
    model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
    if train:
        return model, (train_x, train_y), (test_x, test_y)
    else:
        return model, (vocab, chunk_tags) 

def create_model(train=True):
    if train:
        (train_x, train_y), (test_x, test_y), (vocab, chunk_tags) = process_data.load_data()
    else:
        with open('model/config.pkl', 'rb') as inp:
            (vocab, chunk_tags) = pickle.load(inp)
    model = Sequential()
    model.add(Embedding(len(vocab), EMBED_DIM, mask_zero=True))  # Random embedding
    model.add(Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True)))
    crf = CRF(len(chunk_tags), sparse_target=True)
    model.add(crf)
    model.summary()
    model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
    if train:
        return model, (train_x, train_y), (test_x, test_y)
    else:
        return model, (vocab, chunk_tags) 

def predict_model():
    x_test, original = vocab_test(test_path, vocab, length)
    ws = open(temp_path, mode='w', newline='\n')
    tags = dict(zip(tag.values(), tag.keys()))
    custom_objects = {'CRF': CRF, 'crf_loss': crf.crf_loss, 'crf_viterbi_accuracy': crf.crf_viterbi_accuracy}
    model = load_model(model_path, custom_objects=custom_objects)
    for question, tests in zip(original, x_test):
        raw = model.predict([[tests]])[0][-len(question):]
        result = [np.argmax(row) for row in raw]
        answer = tuple(map(lambda x: tags[x], result))
        ma = map(lambda x: x[0] + '\t' + x[1] + '\n', zip(question, answer))
        ws.writelines(ma)
        ws.write('\n')
    ws.flush()
    ws.close() 

def build(self):
        """Ner模型
        """
        x_in = Input(shape=(self.max_len,), name="Origin-Input-Token")
        s_in = Input(shape=(self.max_len,), name="Origin-Input-Segment")
        x = self.bert_model([x_in, s_in])
        x = Lambda(lambda X: X[:, 1:], name="Ignore-CLS")(x)
        x = self._task_layers(x)
        x = CRF(self.numb_tags, sparse_target=True, name="CRF")(x)
        model = Model([x_in, s_in], x)
        return model 

def build_trained_model(args):
    """模型加载流程
    """
    # 环境设置
    os.environ["CUDA_VISIBLE_DEVICES"] = args.device_map if args.device_map != "cpu" else ""
    # 处理流程
    tokenizer = Tokenizer(args.bert_vocab)
    with codecs.open(os.path.join(args.file_path, "tag_to_id.pkl"), "rb") as f:
        tag_to_id = pickle.load(f)
    with codecs.open(os.path.join(args.file_path, "id_to_tag.pkl"), "rb") as f:
        id_to_tag = pickle.load(f)
    crf_accuracy = CrfAcc(tag_to_id, args.tag_padding).crf_accuracy
    crf_loss = CrfLoss(tag_to_id, args.tag_padding).crf_loss
    custom_objects = {
        "MultiHeadAttention": MultiHeadAttention,
        "LayerNormalization": LayerNormalization,
        "PositionEmbedding": PositionEmbedding,
        "FeedForward": FeedForward,
        "EmbeddingDense": EmbeddingDense,
        "CRF": CRF,
        "crf_accuracy": crf_accuracy,
        "crf_loss": crf_loss,
        "gelu_erf": gelu_erf,
        "gelu_tanh": gelu_tanh,
        "gelu": gelu_erf}
    model = load_model(args.model_path, custom_objects=custom_objects)
    model._make_predict_function()
    viterbi_decoder = Viterbi(model, len(id_to_tag))

    return tokenizer, id_to_tag, viterbi_decoder 

def get_model_cnn_crf(lr=0.001):
    nclass = 5

    seq_input = Input(shape=(None, 3000, 1))
    base_model = get_base_model()
    # for layer in base_model.layers:
    #     layer.trainable = False
    encoded_sequence = TimeDistributed(base_model)(seq_input)
    encoded_sequence = SpatialDropout1D(rate=0.01)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="relu",
                                                               padding="same")(encoded_sequence))
    encoded_sequence = Dropout(rate=0.05)(Convolution1D(128,
                                                               kernel_size=3,
                                                               activation="linear",
                                                               padding="same")(encoded_sequence))

    #out = TimeDistributed(Dense(nclass, activation="softmax"))(encoded_sequence)
    # out = Convolution1D(nclass, kernel_size=3, activation="linear", padding="same")(encoded_sequence)

    crf = CRF(nclass, sparse_target=True)

    out = crf(encoded_sequence)


    model = models.Model(seq_input, out)

    model.compile(optimizers.Adam(lr), crf.loss_function, metrics=[crf.accuracy])
    model.summary()

    return model 

def sl_output_logits(x, nb_classes, use_crf=True):
    if use_crf:
        crf = CRF(nb_classes, sparse_target=False)
        loss = crf.loss_function
        acc = [crf.accuracy]
        outputs = crf(x)
    else:
        loss = 'categorical_crossentropy'
        acc = ['acc']
        outputs = Dense(nb_classes, activation='softmax')(x)
    return outputs, loss, acc 

def load_train(self, config: BuildModelConfig, model_config_path: str=None, model_weights_path: str=None):
        with open(model_config_path, "r", encoding='utf-8') as f:
            if config.use_crf:
                from keras_contrib.layers import CRF
                custom_objects = {'ReversedLSTM': ReversedLSTM, 'CRF': CRF}
                self.train_model = model_from_yaml(f.read(), custom_objects=custom_objects)
            else:
                custom_objects = {'ReversedLSTM': ReversedLSTM}
                self.train_model = model_from_yaml(f.read(), custom_objects=custom_objects)
        self.train_model.load_weights(model_weights_path)

        loss = {}
        metrics = {}
        if config.use_crf:
            out_layer_name = 'crf'
            offset = 0
            if config.use_pos_lm:
                offset += 2
            if config.use_word_lm:
                offset += 2
            loss[out_layer_name] = self.train_model.layers[-1-offset].loss_function
            metrics[out_layer_name] = self.train_model.layers[-1-offset].accuracy
        else:
            out_layer_name = 'main_pred'
            loss[out_layer_name] = 'sparse_categorical_crossentropy'
            metrics[out_layer_name] = 'accuracy'

        if config.use_pos_lm:
            prev_layer_name = 'shifted_pred_prev'
            next_layer_name = 'shifted_pred_next'
            loss[prev_layer_name] = loss[next_layer_name] = 'sparse_categorical_crossentropy'
            metrics[prev_layer_name] = metrics[next_layer_name] = 'accuracy'
        self.train_model.compile(Adam(clipnorm=5.), loss=loss, metrics=metrics)

        self.eval_model = Model(inputs=self.train_model.inputs, outputs=self.train_model.outputs[0]) 

def load_eval(self, config: BuildModelConfig, eval_model_config_path: str,
                  eval_model_weights_path: str) -> None:
        with open(eval_model_config_path, "r", encoding='utf-8') as f:
            if config.use_crf:
                from keras_contrib.layers import CRF
                custom_objects = {'ReversedLSTM': ReversedLSTM, 'CRF': CRF}
                self.eval_model = model_from_yaml(f.read(), custom_objects=custom_objects)
            else:
                custom_objects = {'ReversedLSTM': ReversedLSTM}
                self.eval_model = model_from_yaml(f.read(), custom_objects=custom_objects)
        self.eval_model.load_weights(eval_model_weights_path)
        self.eval_model._make_predict_function() 

def __build_model(self):
        assert self.max_depth >= 1, "The parameter max_depth is at least 1"

        src_seq_input = Input(shape=(self.max_seq_len,), dtype="int32", name="src_seq_input")
        mask = Lambda(lambda x: padding_mask(x, x))(src_seq_input)

        emb_output = self.__input(src_seq_input)
        enc_output = self.__encoder(emb_output, mask)

        if self.use_crf:
            crf = CRF(self.tgt_vocab_size + 1, sparse_target=self.sparse_target)
            y_pred = crf(self.__output(enc_output))
        else:
            y_pred = self.__output(enc_output)

        model = Model(inputs=[src_seq_input], outputs=[y_pred])
        parallel_model = model
        if self.num_gpu > 1:
            parallel_model = multi_gpu_model(model, gpus=self.num_gpu)

        if self.use_crf:
            parallel_model.compile(self.optimizer, loss=crf_loss, metrics=[crf_accuracy])
        else:
            confidence_penalty = K.mean(
                self.confidence_penalty_weight *
                K.sum(y_pred * K.log(y_pred), axis=-1))
            model.add_loss(confidence_penalty)
            parallel_model.compile(optimizer=self.optimizer, loss=categorical_crossentropy, metrics=['accuracy'])

        return model, parallel_model 

def build_model(token_num,
                tag_num,
                embedding_dim=100,
                embedding_weights=None,
                rnn_units=100,
                return_attention=False,
                lr=1e-3):
    """Build the model for predicting tags.

    :param token_num: Number of tokens in the word dictionary.
    :param tag_num: Number of tags.
    :param embedding_dim: The output dimension of the embedding layer.
    :param embedding_weights: Initial weights for embedding layer.
    :param rnn_units: The number of RNN units in a single direction.
    :param return_attention: Whether to return the attention matrix.
    :param lr: Learning rate of optimizer.
    :return model: The built model.
    """
    if embedding_weights is not None and not isinstance(embedding_weights, list):
        embedding_weights = [embedding_weights]

    input_layer = keras.layers.Input(shape=(None,))
    embd_layer = keras.layers.Embedding(input_dim=token_num,
                                        output_dim=embedding_dim,
                                        mask_zero=True,
                                        weights=embedding_weights,
                                        trainable=embedding_weights is None,
                                        name='Embedding')(input_layer)
    lstm_layer = keras.layers.Bidirectional(keras.layers.LSTM(units=rnn_units,
                                                              recurrent_dropout=0.4,
                                                              return_sequences=True),
                                            name='Bi-LSTM')(embd_layer)
    attention_layer = Attention(attention_activation='sigmoid',
                                attention_width=9,
                                return_attention=return_attention,
                                name='Attention')(lstm_layer)
    if return_attention:
        attention_layer, attention = attention_layer
    crf = CRF(units=tag_num, sparse_target=True, name='CRF')

    outputs = [crf(attention_layer)]
    loss = {'CRF': crf.loss_function}
    if return_attention:
        outputs.append(attention)
        loss['Attention'] = Attention.loss(1e-4)

    model = keras.models.Model(inputs=input_layer, outputs=outputs)
    model.compile(
        optimizer=keras.optimizers.Adam(lr=lr),
        loss=loss,
        metrics={'CRF': crf.accuracy},
    )
    return model 

def build(self):
        inputs = [] #Create input for Model

        # build word embeddings
        input_words = Input(shape=(None,), dtype='int32', name='word_ids')
        inputs.append(input_words)
        if self.config.embeddings is None:
            word_embeddings = Embedding(input_dim=self.config.nwords,
                                        output_dim=self.config.dim_word,
                                        mask_zero=True,
                                        name="word_embeddings")(input_words)
        else:
            word_embeddings = Embedding(input_dim=self.config.nwords,
                                        output_dim=self.config.dim_word,
                                        mask_zero=True,
                                        weights=[self.config.embeddings],
                                        trainable=self.config.train_embeddings,
                                        name="word_embeddings")(input_words)

        # build character based word embedding
        if self.config.use_chars:
            input_chars = Input(batch_shape=(None, None, None), dtype='int32', name='char_ids')
            inputs.append(input_chars)
            char_embeddings = Embedding(input_dim=self.config.nchars,
                                        output_dim=self.config.dim_char,
                                        mask_zero=True,
                                        name='char_embeddings')(input_chars)
            s = K.shape(char_embeddings)
            char_embeddings = Lambda(lambda x: K.reshape(x, shape=(-1, s[-2], self.config.dim_char)))(char_embeddings)

            # BiLSTM for char_embeddings
            fwd_state = LSTM(self.config.hidden_size_char, return_state=True, name='fw_char_lstm')(char_embeddings)[-2]
            bwd_state = LSTM(self.config.hidden_size_char, return_state=True, go_backwards=True, name='bw_char_lstm')(char_embeddings)[-2]
            char_embeddings = Concatenate(axis=-1)([fwd_state, bwd_state])
            # shape = (batch size, max sentence length, char hidden size)
            char_embeddings = Lambda(lambda x: K.reshape(x, shape=[-1, s[1], 2 * self.config.hidden_size_char]))(char_embeddings)

            #combine characters and word
            word_embeddings = Concatenate(axis=-1)([word_embeddings, char_embeddings])

        word_embeddings = Dropout(self.config.dropout)(word_embeddings)
        encoded_text = Bidirectional(LSTM(units=self.config.hidden_size_lstm, return_sequences=True), name="bidirectional")(word_embeddings)
        encoded_text = Dropout(self.config.dropout)(encoded_text)
        #encoded_text = Dense(100, activation='tanh')(encoded_text)

        if self.config.use_crf:
            crf = CRF(self.config.ntags, sparse_target=False)
            self._loss = crf.loss_function
            pred = crf(encoded_text)

        else:
            self._loss = 'categorical_crossentropy'
            pred = Dense(self.config.ntags, activation='softmax')(encoded_text)

        self.model = Model(inputs, pred)