org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper Java Examples

The following examples show how to use org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LuceneRecord.java    From HongsCORE with MIT License 5 votes vote down vote up
/**
 * 存储分析器
 * @return
 * @throws HongsException
 */
protected Analyzer getAnalyzer() throws HongsException {
    /*Default*/ Analyzer  ad = new StandardAnalyzer();
    Map<String, Analyzer> az = new HashMap();
    Map<String, Map     > fs = getFields(  );
    for(Map.Entry<String, Map> et : fs.entrySet()) {
        String fn = et.getKey(  );
        Map    fc = et.getValue();
        if (srchable(fc)) {
            // 注意: 搜索对应的字段名开头为 $
            az.put("$" + fn, getAnalyzer(fc) );
        }
    }
    return new PerFieldAnalyzerWrapper(ad, az);
}
 
Example #2
Source File: VocabularyIndexAnalyzer.java    From SciGraph with Apache License 2.0 5 votes vote down vote up
public VocabularyIndexAnalyzer() throws IOException, URISyntaxException {
  super(NO_REUSE_STRATEGY);
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example #3
Source File: VocabularyQueryAnalyzer.java    From SciGraph with Apache License 2.0 5 votes vote down vote up
public VocabularyQueryAnalyzer() {
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example #4
Source File: AnalysisUtilsTest.java    From stratio-cassandra with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() {
    englishAnalyzer = new EnglishAnalyzer();
    spanishAnalyzer = new SpanishAnalyzer();
    Map<String, Analyzer> analyzers = new HashMap<>();
    analyzers.put("english", englishAnalyzer);
    analyzers.put("spanish", spanishAnalyzer);
    perFieldAnalyzer = new PerFieldAnalyzerWrapper(spanishAnalyzer, analyzers);
}
 
Example #5
Source File: TripleIndexCreator.java    From AGDISTIS with GNU Affero General Public License v3.0 5 votes vote down vote up
public void createIndex(List<File> files, String idxDirectory, String baseURI) {
	try {
		urlAnalyzer = new SimpleAnalyzer(LUCENE_VERSION);
		literalAnalyzer = new LiteralAnalyzer(LUCENE_VERSION);
		Map<String, Analyzer> mapping = new HashMap<String, Analyzer>();
		mapping.put(TripleIndex.FIELD_NAME_SUBJECT, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_PREDICATE, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_OBJECT_URI, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_OBJECT_LITERAL, literalAnalyzer);
		PerFieldAnalyzerWrapper perFieldAnalyzer = new PerFieldAnalyzerWrapper(urlAnalyzer, mapping);

		File indexDirectory = new File(idxDirectory);
		indexDirectory.mkdir();
		directory = new MMapDirectory(indexDirectory);
		IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, perFieldAnalyzer);
		iwriter = new IndexWriter(directory, config);
		iwriter.commit();
		for (File file : files) {
			String type = FileUtil.getFileExtension(file.getName());
			if (type.equals(TTL))
				indexTTLFile(file, baseURI);
			if (type.equals(TSV))
				indexTSVFile(file);
			iwriter.commit();
		}
		iwriter.close();
		ireader = DirectoryReader.open(directory);
	} catch (Exception e) {
		log.error("Error while creating TripleIndex.", e);
	}
}
 
Example #6
Source File: TripleIndexCreatorContext.java    From AGDISTIS with GNU Affero General Public License v3.0 5 votes vote down vote up
public void createIndex(List<File> files, String idxDirectory, String baseURI) {
	try {
		urlAnalyzer = new SimpleAnalyzer(LUCENE_VERSION);
		literalAnalyzer = new LiteralAnalyzer(LUCENE_VERSION);
		Map<String, Analyzer> mapping = new HashMap<String, Analyzer>();
		mapping.put(FIELD_NAME_URI, urlAnalyzer);
		mapping.put(FIELD_NAME_SURFACE_FORM, literalAnalyzer);
		mapping.put(FIELD_NAME_URI_COUNT, literalAnalyzer);
		mapping.put(FIELD_NAME_CONTEXT, literalAnalyzer);
		PerFieldAnalyzerWrapper perFieldAnalyzer = new PerFieldAnalyzerWrapper(urlAnalyzer, mapping);

		File indexDirectory = new File(idxDirectory);
		indexDirectory.mkdir();
		directory = new MMapDirectory(indexDirectory);
		IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, perFieldAnalyzer);
		iwriter = new IndexWriter(directory, config);
		iwriter.commit();
		for (File file : files) {
			String type = FileUtil.getFileExtension(file.getName());
			if (type.equals(TTL))
				indexTTLFile(file, baseURI);
			iwriter.commit();
		}
	} catch (Exception e) {
		log.error("Error while creating TripleIndex.", e);
	}
}
 
Example #7
Source File: MtasDocumentIndex.java    From inception with Apache License 2.0 4 votes vote down vote up
private synchronized IndexWriter getIndexWriter() throws IOException
{
    if (_indexWriter == null) {
        log.debug("Opening index for project [{}]({})", project.getName(), project.getId());

        OPEN_INDEXES.put(project.getId(), this);
        
        // Initialize and populate the hash maps for the layers and features
        features = schemaService.listAnnotationFeature(project).stream()
                .filter(feat -> feat.getLayer().isEnabled())
                .filter(feat -> feat.isEnabled())
                .collect(Collectors.toList());
        
        // Add the project id to the configuration
        JSONObject jsonParserConfiguration = new JSONObject();
        jsonParserConfiguration.put(PARAM_PROJECT_ID, project.getId());
        
        // Tokenizer parameters
        Map<String, String> tokenizerArguments = new HashMap<>();
        tokenizerArguments.put(ARGUMENT_PARSER, MtasUimaParser.class.getName());
        tokenizerArguments.put(ARGUMENT_PARSER_ARGS, jsonParserConfiguration.toString());
        
        // Build analyzer
        Analyzer mtasAnalyzer = CustomAnalyzer.builder()
                .withTokenizer(MtasTokenizerFactory.class, tokenizerArguments)
                .build();
        
        Map<String, Analyzer> analyzerPerField = new HashMap<String, Analyzer>();
        analyzerPerField.put(FIELD_CONTENT, mtasAnalyzer);
        
        PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(),
                analyzerPerField);
        
        // Build IndexWriter
        FileUtils.forceMkdir(getIndexDir());
        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        config.setCodec(Codec.forName(MTAS_CODEC_NAME));
        IndexWriter indexWriter = new IndexWriter(FSDirectory.open(getIndexDir().toPath()),
                config);
        
        // Initialize the index
        indexWriter.commit();
        
        // After the index has been initialized, assign the _indexWriter - this is also used
        // by isOpen() to check if the index writer is available.
        _indexWriter = indexWriter;
    }
    
    return _indexWriter;
}
 
Example #8
Source File: Message.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public static Analyzer createLuceneAnalyzer() {
  Map<String, Analyzer> map = new HashMap<>();
  map.put("from", new UAX29URLEmailAnalyzer());
  return new PerFieldAnalyzerWrapper(new StandardAnalyzer(), map);
}
 
Example #9
Source File: TreatmentCurator.java    From hmftools with GNU General Public License v3.0 4 votes vote down vote up
@NotNull
private static Analyzer indexAnalyzer() {
    Map<String, Analyzer> fieldAnalyzers = Maps.newHashMap();
    fieldAnalyzers.put(DRUG_NAME_FIELD, concatenatingAnalyzer());
    return new PerFieldAnalyzerWrapper(wordDelimiterAnalyzer(), fieldAnalyzers);
}
 
Example #10
Source File: MtasSearchTestConsistency.java    From mtas with Apache License 2.0 4 votes vote down vote up
/**
 * Creates the index.
 *
 * @param configFile the config file
 * @param files the files
 * @throws IOException Signals that an I/O exception has occurred.
 */
private static void createIndex(String configFile,
    HashMap<String, String> files) throws IOException {
  // analyzer
  Map<String, String> paramsCharFilterMtas = new HashMap<>();
  paramsCharFilterMtas.put("type", "file");
  Map<String, String> paramsTokenizer = new HashMap<>();
  paramsTokenizer.put("configFile", configFile);
  Analyzer mtasAnalyzer = CustomAnalyzer
      .builder(Paths.get("docker").toAbsolutePath())
      .addCharFilter("mtas", paramsCharFilterMtas)
      .withTokenizer("mtas", paramsTokenizer).build();
  Map<String, Analyzer> analyzerPerField = new HashMap<>();
  analyzerPerField.put(FIELD_CONTENT, mtasAnalyzer);
  PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(
      new StandardAnalyzer(), analyzerPerField);
  // indexwriter
  IndexWriterConfig config = new IndexWriterConfig(analyzer);
  config.setUseCompoundFile(false);
  config.setCodec(Codec.forName("MtasCodec"));
  IndexWriter w = new IndexWriter(directory, config);
  // delete
  w.deleteAll();
  // add
  int counter = 0;
  for (Entry<String, String> entry : files.entrySet()) {
    addDoc(w, counter, entry.getKey(), entry.getValue());
    if (counter == 0) {
      w.commit();
    } else {
      addDoc(w, counter, entry.getKey(), entry.getValue());
      addDoc(w, counter, "deletable", entry.getValue());
      w.commit();
      w.deleteDocuments(new Term(FIELD_ID, Integer.toString(counter)));
      w.deleteDocuments(new Term(FIELD_TITLE, "deletable"));
      addDoc(w, counter, entry.getKey(), entry.getValue());
    }
    counter++;
  }
  w.commit();
  // finish
  w.close();
}
 
Example #11
Source File: LumongoIndex.java    From lumongo with Apache License 2.0 4 votes vote down vote up
public PerFieldAnalyzerWrapper getPerFieldAnalyzer() throws Exception {
	return lumongoAnalyzerFactory.getPerFieldAnalyzer();
}
 
Example #12
Source File: IndexSegmentInterface.java    From lumongo with Apache License 2.0 votes vote down vote up
PerFieldAnalyzerWrapper getPerFieldAnalyzer() throws Exception;