org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess Java Examples

protected static TokenizerFactory getTokenizerFactory(VectorsConfiguration configuration) {
    if (configuration == null)
        return null;

    if (configuration.getTokenizerFactory() != null && !configuration.getTokenizerFactory().isEmpty()) {
        try {
            TokenizerFactory factory =
                            (TokenizerFactory) Class.forName(configuration.getTokenizerFactory()).newInstance();

            if (configuration.getTokenPreProcessor() != null && !configuration.getTokenPreProcessor().isEmpty()) {
                TokenPreProcess preProcessor =
                                (TokenPreProcess) Class.forName(configuration.getTokenPreProcessor()).newInstance();
                factory.setTokenPreProcessor(preProcessor);
            }

            return factory;

        } catch (Exception e) {
            log.error("Can't instantiate saved TokenizerFactory: {}", configuration.getTokenizerFactory());
        }
    }
    return null;
}
 

@Test
public void testPreProcessor() {
    TokenPreProcess preProcess = new EndingPreProcessor();
    String endingTest = "ending";
    assertEquals("end", preProcess.preProcess(endingTest));

}
 

@Override
public String preProcess(String token) {
    String s = token;
    for(TokenPreProcess tpp : preProcessors){
        s = tpp.preProcess(s);
    }
    return s;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
    return this.preProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
    this.tokenPreProcess = preProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcess) {
    this.preProcess = tokenPreProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
  this.tokenPreProcess = tokenPreProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
    this.preProcess = preProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
    this.preProcess = tokenPreProcessor;
}
 

@Override
public void setTokenPreProcessor(@NonNull TokenPreProcess tokenPreProcessor) {
    this.preProcessor = tokenPreProcessor;
}
 

public EmbeddedStemmingPreprocessor(@NonNull TokenPreProcess preProcess) {
    this.preProcessor = preProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcess) {
    this.tokenPreProcess = tokenPreProcess;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
    return tokenPreProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
    this.tokenPreProcess = tokenPreProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
    this.tokenPreProcess = preProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
    this.preProcess = preProcessor;
}
 

/**
 * @param vocab                   Vocabulary, as a navigable map
 * @param preTokenizePreProcessor The preprocessor that should be used on the raw strings, before splitting
 */
public BertWordPieceTokenizerFactory(NavigableMap<String, Integer> vocab, TokenPreProcess preTokenizePreProcessor) {
    this.vocab = vocab;
    this.preTokenizePreProcessor = preTokenizePreProcessor;
}
 

public CompositePreProcessor(@NonNull TokenPreProcess... preProcessors){
    Preconditions.checkState(preProcessors.length > 0, "No preprocessors were specified (empty input)");
    this.preProcessors = Arrays.asList(preProcessors);
}
 

public CompositePreProcessor(@NonNull Collection<? extends TokenPreProcess> preProcessors){
    Preconditions.checkState(!preProcessors.isEmpty(), "No preprocessors were specified (empty input)");
    this.preProcessors = new ArrayList<>(preProcessors);
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
    this.preProcessor = preProcessor;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
    return this.preProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
    this.preProcessor = tokenPreProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
  this.tokenPreProcess = tokenPreProcessor;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
  return tokenPreProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
  this.tokenPreProcess = preProcessor;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
  return tokenPreProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
  this.tokenPreProcess = preProcessor;
}
 

@Override
public TokenPreProcess getTokenPreProcessor() {
  return tokenPreProcess;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess preProcessor) {
  this.tokenPreProcess = preProcessor;
}
 

@Override
public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
  this.tokenPreProcess = tokenPreProcessor;
}