org.apache.lucene.analysis.core.KeywordAnalyzer Java Examples

The following examples show how to use org.apache.lucene.analysis.core.KeywordAnalyzer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlurUtilsTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #2
Source File: LuceneOrderedDocCollectorTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testSearchWithScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, true);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(1.0F));
    assertThat(values.next().get(0), Matchers.is(1.0F));
}
 
Example #3
Source File: LuceneOrderedDocCollectorTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testSearchNoScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, false);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(Float.NaN));
    assertThat(values.next().get(0), Matchers.is(Float.NaN));
}
 
Example #4
Source File: BaseFieldManagerTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
protected FieldManager newFieldManager(boolean create) throws IOException {
  return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer(), new Configuration()) {
    @Override
    protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) {
      return true;
    }

    @Override
    protected void tryToLoad(String field) {

    }

    @Override
    protected List<String> getFieldNamesToLoad() throws IOException {
      return new ArrayList<String>();
    }
  };
}
 
Example #5
Source File: HdfsDirectorySymlinkTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testSymlinkWithIndexes() throws IOException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_base, "dir1"));
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer1 = new IndexWriter(dir1, conf.clone());
  writer1.addDocument(getDoc());
  writer1.close();

  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_base, "dir2"));
  IndexWriter writer2 = new IndexWriter(dir2, conf.clone());
  writer2.addIndexes(dir1);
  writer2.close();

  DirectoryReader reader1 = DirectoryReader.open(dir1);
  DirectoryReader reader2 = DirectoryReader.open(dir2);

  assertEquals(1, reader1.maxDoc());
  assertEquals(1, reader2.maxDoc());
  assertEquals(1, reader1.numDocs());
  assertEquals(1, reader2.numDocs());

  Document document1 = reader1.document(0);
  Document document2 = reader2.document(0);

  assertEquals(document1.get("id"), document2.get("id"));
}
 
Example #6
Source File: FastHdfsKeyValueDirectoryTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example #7
Source File: BlurUtilsTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private IndexReader getReader() throws CorruptIndexException, LockObtainFailedException, IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f1", Store.YES));

  Document doc1 = new Document();
  doc.add(new StringField("a", "b", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #8
Source File: TableShardCountCollapserTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
    throws IOException {
  HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
  mergePolicy.setUseCompoundFile(false);
  IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);

  Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
  int partition = partitioner.getPartition(new IntWritable(i), null, totalShardCount);
  assertEquals(i, partition);

  Document doc = getDoc(i);
  indexWriter.addDocument(doc);
  indexWriter.close();
}
 
Example #9
Source File: MutatableActionTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  TableContext.clear();
  _base = new File(TMPDIR, "MutatableActionTest");
  rmr(_base);

  File file = new File(_base, TABLE);
  file.mkdirs();

  TableContext.clear();
  TableDescriptor tableDescriptor = new TableDescriptor();
  tableDescriptor.setName("test");
  tableDescriptor.setTableUri(file.toURI().toString());
  TableContext tableContext = TableContext.create(tableDescriptor);
  ShardContext shardContext = ShardContext.create(tableContext, "test");
  _action = new MutatableAction(shardContext);
  _conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
}
 
Example #10
Source File: VocabularyNeo4jImpl.java    From SciGraph with Apache License 2.0 6 votes vote down vote up
@Inject
public VocabularyNeo4jImpl(GraphDatabaseService graph,
    @Nullable @IndicatesNeo4jGraphLocation String neo4jLocation, CurieUtil curieUtil,
    NodeTransformer transformer) throws IOException {
  this.graph = graph;
  this.curieUtil = curieUtil;
  this.transformer = transformer;
  if (null != neo4jLocation) {
    Directory indexDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/node/node_auto_index"))
            .toPath());
    Directory spellDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/spellchecker"))
            .toPath());
    spellChecker = new SpellChecker(spellDirectory);
    try (IndexReader reader = DirectoryReader.open(indexDirectory)) {
      IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer());
      spellChecker.indexDictionary(new LuceneDictionary(reader, NodeProperties.LABEL
          + LuceneUtils.EXACT_SUFFIX), config, true);
    }
  } else {
    spellChecker = null;
  }
}
 
Example #11
Source File: WordFrequencyStore.java    From SourcererCC with GNU General Public License v3.0 6 votes vote down vote up
public void prepareIndex() throws IOException {
    File globalWFMDIr = new File(Util.GTPM_INDEX_DIR);
    if (!globalWFMDIr.exists()) {
        Util.createDirs(Util.GTPM_INDEX_DIR);
    }
    KeywordAnalyzer keywordAnalyzer = new KeywordAnalyzer();
    IndexWriterConfig wfmIndexWriterConfig = new IndexWriterConfig(Version.LUCENE_46, keywordAnalyzer);
    wfmIndexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
    wfmIndexWriterConfig.setRAMBufferSizeMB(1024);

    logger.info("PREPARE INDEX");
    try {
        wfmIndexWriter = new IndexWriter(FSDirectory.open(new File(Util.GTPM_INDEX_DIR)), wfmIndexWriterConfig);
        wfmIndexWriter.commit();
        wfmIndexer = new DocumentMaker(wfmIndexWriter);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example #12
Source File: CodeSearcher.java    From SourcererCC with GNU General Public License v3.0 6 votes vote down vote up
public CodeSearcher(String indexDir, String field) {
    logger.info("index directory: "+ indexDir);
    this.field = field;
    this.indexDir = indexDir;
    try {
        this.reader = DirectoryReader.open(FSDirectory.open(new File(
                this.indexDir)));
    } catch (IOException e) {
        logger.error("cant get the reader to index dir, exiting, "
                + indexDir);
        e.printStackTrace();
        System.exit(1);
    }
    this.searcher = new IndexSearcher(this.reader);
    this.analyzer = new KeywordAnalyzer();//
            //new WhitespaceAnalyzer(Version.LUCENE_46); // TODO: pass
                                                               // the
                                                               // analyzer
                                                               // as
                                                               // argument
                                                               // to
                                                               // constructor
    new CloneHelper(); // i don't remember why we are making this object?
    this.queryParser = new QueryParser(Version.LUCENE_46, this.field,
            analyzer);
}
 
Example #13
Source File: TestConfigurableAnalyzerFactory.java    From database with GNU General Public License v2.0 6 votes vote down vote up
@Override
String[] getExtraProperties() {
	String analyzer = ConfigurableAnalyzerFactory.Options.ANALYZER;
	return new String[]{
	FullTextIndex.Options.ANALYZER_FACTORY_CLASS, ConfigurableAnalyzerFactory.class.getName(),
	analyzer+"_."+AnalyzerOptions.LIKE, "x-empty",
	analyzer+"x-empty."+AnalyzerOptions.ANALYZER_CLASS, EmptyAnalyzer.class.getName(),
	analyzer+"x-terms."+AnalyzerOptions.PATTERN, "\\W+",
	analyzer+"x-splits."+AnalyzerOptions.ANALYZER_CLASS, TermCompletionAnalyzer.class.getName(),
	analyzer+"x-splits."+AnalyzerOptions.STOPWORDS, AnalyzerOptions.STOPWORDS_VALUE_NONE,
	analyzer+"x-splits."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-splits."+AnalyzerOptions.SUB_WORD_BOUNDARY, "(?<!\\p{L}|\\p{N})(?=\\p{L}|\\p{N})|(?<!\\p{Lu})(?=\\p{Lu})|(?<=\\p{N})(?=\\p{L})",
	analyzer+"x-hyphen."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "false",
	analyzer+"x-hyphen2."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen2."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen2."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen2."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "true",
	analyzer+"x-keywords."+AnalyzerOptions.ANALYZER_CLASS, KeywordAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.ANALYZER_CLASS, StandardAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.STOPWORDS, GermanAnalyzer.class.getName(),
	};
}
 
Example #14
Source File: PrimeDocOverFlowHelper.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static Directory getDirectoryUpdateRow(String currentRowId) {
  try {
    RAMDirectory directoryUpdateRow = new RAMDirectory();
    IndexWriter writer = new IndexWriter(directoryUpdateRow, new IndexWriterConfig(Version.LUCENE_43,
        new KeywordAnalyzer()));
    Document document = new Document();
    document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
    document.add(new StringField(BlurConstants.UPDATE_ROW, currentRowId, Store.NO));
    writer.addDocument(document);
    writer.close();
    return directoryUpdateRow;
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example #15
Source File: SecureAtomicReaderTestBase.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private DirectoryReader createReader() throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  Directory dir = new RAMDirectory();
  IndexWriter writer = new IndexWriter(dir, conf);
  AccessControlWriter accessControlWriter = getAccessControlFactory().getWriter();
  addDoc(writer, accessControlWriter, "r1", "d1", 0);
  addDoc(writer, accessControlWriter, "r2", "d1", 1);
  addDoc(writer, accessControlWriter, "r1", "d2", 2);
  addDoc(writer, accessControlWriter, "r2", "d2", 3);
  addDoc(writer, accessControlWriter, "r1", "d1", 4, "test");
  addDoc(writer, accessControlWriter, "r1", "d1", 5, "termmask");
  writer.close();

  return DirectoryReader.open(dir);
}
 
Example #16
Source File: TestingPagingCollector.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static IndexReader getReaderFlatScore(int length) throws Exception {
  _directory = new RAMDirectory();
  IndexWriter indexWriter = new IndexWriter(_directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()));
  for (int i = 0; i < length; i++) {
    Document document = new Document();
    document.add(new StringField("f1", "value", Store.NO));
    document.add(new IntDocValuesField("index", i));
    document.add(new IntField("index", i, Store.YES));
    indexWriter.addDocument(document);
  }
  indexWriter.close();
  return DirectoryReader.open(_directory);
}
 
Example #17
Source File: FilterCacheTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private void writeDocs(FilterCache filterCache, RAMDirectory directory) throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  conf.setCodec(new Blur024Codec());
  IndexWriter indexWriter = new IndexWriter(directory, conf);
  int count = 10000;
  addDocs(indexWriter, count);
  indexWriter.close();
}
 
Example #18
Source File: FieldTypePluginLoader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) {
  if (queryAnalyzer == null) return null;

  if (!(queryAnalyzer instanceof TokenizerChain)) {
    return new KeywordAnalyzer();
  }

  return ((TokenizerChain) queryAnalyzer).getMultiTermAnalyzer();
}
 
Example #19
Source File: CacheDirectoryTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Test
public void test3() throws IOException, InterruptedException {
  // Thread.sleep(30000);
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(_cacheDirectory, conf);
  int docs = 100000;
  for (int i = 0; i < docs; i++) {
    if (i % 500 == 0) {
      System.out.println(i);
    }
    writer.addDocument(newDoc());
    // Thread.sleep(1);
  }
  writer.close();
  System.out.println("done writing");

  DirectoryReader reader = DirectoryReader.open(_cacheDirectory);
  System.out.println("done opening");
  assertEquals(docs, reader.numDocs());

  Document document = reader.document(0);
  System.out.println("done fetching");
  System.out.println(document);

  IndexSearcher searcher = new IndexSearcher(reader);
  TopDocs topDocs = searcher.search(new TermQuery(new Term("test", "test")), 10);
  System.out.println("done searching");
  assertEquals(docs, topDocs.totalHits);

  reader.close();
}
 
Example #20
Source File: BlurIndexWriterTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private Directory addDir(String v) throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, config);
  writer.addDocument(getDoc(v));
  writer.close();

  return directory;
}
 
Example #21
Source File: BlurSecureIndexSearcherTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private IndexReader getIndexReader() throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  Directory dir = new RAMDirectory();
  IndexWriter writer = new IndexWriter(dir, conf);
  writer.close();
  return DirectoryReader.open(dir);
}
 
Example #22
Source File: LoadTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static void createIndex(Directory directory, AccessControlFactory accessControlFactory) throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);

  AccessControlWriter accessControlWriter = accessControlFactory.getWriter();
  Random random = new Random(1);
  for (long i = 0; i < MAX_DOCS; i++) {
    if (i % 1000000 == 0) {
      System.out.println("Building " + i);
    }
    writer.addDocument(accessControlWriter.addDiscoverVisiblity("d1",
        accessControlWriter.addReadVisiblity("r1", getDoc(i, random))));
  }
  writer.close();
}
 
Example #23
Source File: QueryFactory.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private QueryExpression create(Request request, ResourceDefinition resourceDefinition) throws InvalidQueryException {
    String queryString;
    if (request.getCardinality() == Request.Cardinality.INSTANCE) {
        String idPropertyName = resourceDefinition.getIdPropertyName();
        queryString = String.format("%s:%s", idPropertyName, request.<String>getProperty(idPropertyName));
    } else {
        queryString = request.getQueryString();
    }

    QueryExpression queryExpression;
    if (queryString != null && !queryString.isEmpty()) {
        QueryParser queryParser = new QueryParser(Version.LUCENE_48, "name", new KeywordAnalyzer());
        queryParser.setLowercaseExpandedTerms(false);
        queryParser.setAllowLeadingWildcard(true);
        Query query;
        try {
            query = queryParser.parse((String) escape(queryString));
        } catch (ParseException e) {
            throw new InvalidQueryException(e.getMessage());
        }
        LOG.info("LuceneQuery: {}", query);
        queryExpression = create(query, resourceDefinition);
    } else {
        queryExpression = new AlwaysQueryExpression();
    }
    // add query properties to request so that they are returned
    request.addAdditionalSelectProperties(queryExpression.getProperties());
    return queryExpression;
}
 
Example #24
Source File: HdfsFieldManagerTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Override
protected BaseFieldManager newFieldManager(boolean create) throws IOException {
  Configuration config = new Configuration();
  Path path = new Path(DFS_FIELD_MANAGER_PATH);
  FileSystem fileSystem = path.getFileSystem(config);
  if (create) {
    fileSystem.delete(path, true);
  }
  return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config);
}
 
Example #25
Source File: VocabularyQueryAnalyzer.java    From SciGraph with Apache License 2.0 5 votes vote down vote up
public VocabularyQueryAnalyzer() {
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example #26
Source File: VocabularyIndexAnalyzer.java    From SciGraph with Apache License 2.0 5 votes vote down vote up
public VocabularyIndexAnalyzer() throws IOException, URISyntaxException {
  super(NO_REUSE_STRATEGY);
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example #27
Source File: SolrSchemaUtil.java    From jesterj with Apache License 2.0 5 votes vote down vote up
private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) {
  if (queryAnalyzer == null) return null;

  if (!(queryAnalyzer instanceof TokenizerChain)) {
    return new KeywordAnalyzer();
  }
  return ((TokenizerChain) queryAnalyzer).getMultiTermAnalyzer();

}
 
Example #28
Source File: DependentTermQueryBuilderTest.java    From querqy with Apache License 2.0 5 votes vote down vote up
@Test
public void testThatResultsAreFound() throws Exception {
    ConstantFieldBoost fieldBoost = new ConstantFieldBoost(1f);
    Analyzer analyzer = new KeywordAnalyzer();

    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, analyzer);

    TestUtil.addNumDocsWithStringField("f1", "v1", indexWriter, 1);
    TestUtil.addNumDocsWithStringField("f1", "v2", indexWriter, 1);

    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader);

    DocumentFrequencyCorrection dfc = new DocumentFrequencyCorrection();

    Term term = new Term("f1", "v1");
    dfc.newClause();
    dfc.prepareTerm(term);
    dfc.finishedUserQuery();

    DependentTermQuery query = new DependentTermQuery(term, dfc, fieldBoost);

    TopDocs topDocs = indexSearcher.search(query, 10);

    assertEquals(1, topDocs.totalHits.value);
    Document resultDoc = indexSearcher.doc(topDocs.scoreDocs[0].doc);
    assertEquals("v1", resultDoc.get("f1"));

    indexReader.close();
    directory.close();
    analyzer.close();

}
 
Example #29
Source File: LuceneQueryBuilderTest.java    From querqy with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    keywordAnalyzer = new KeywordAnalyzer();
    searchFields = new HashMap<>();
    searchFields.put("f1", 1.0f);
    searchFields.put("f11", 1.0f);
    searchFields.put("f12", 1.0f);
    searchFields.put("f13", 1.0f);
    searchFields.put("f14", 1.0f);
    searchFields.put("f15", 1.0f);

    searchFields.put("f2", 2.0f);
    searchFields.put("f21", 2.0f);
    searchFields.put("f22", 2.0f);
    searchFields.put("f23", 2.0f);
    searchFields.put("f24", 2.0f);
    searchFields.put("f25", 2.0f);

    searchFields.put("f3", 3.0f);
    searchFields.put("f31", 3.0f);
    searchFields.put("f32", 3.0f);
    searchFields.put("f33", 3.0f);
    searchFields.put("f34", 3.0f);
    searchFields.put("f35", 3.0f);

    stopWords = new HashSet<>(Arrays.asList("stopA", "stopB", "stopC"));

}
 
Example #30
Source File: MonitorConfiguration.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static IndexWriterConfig defaultIndexWriterConfig() {
  IndexWriterConfig iwc = new IndexWriterConfig(new KeywordAnalyzer());
  TieredMergePolicy mergePolicy = new TieredMergePolicy();
  mergePolicy.setSegmentsPerTier(4);
  iwc.setMergePolicy(mergePolicy);
  iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
  return iwc;
}