Java Code Examples for org.apache.lucene.analysis.core.KeywordAnalyzer

The following examples show how to use org.apache.lucene.analysis.core.KeywordAnalyzer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
String[] getExtraProperties() {
	String analyzer = ConfigurableAnalyzerFactory.Options.ANALYZER;
	return new String[]{
	FullTextIndex.Options.ANALYZER_FACTORY_CLASS, ConfigurableAnalyzerFactory.class.getName(),
	analyzer+"_."+AnalyzerOptions.LIKE, "x-empty",
	analyzer+"x-empty."+AnalyzerOptions.ANALYZER_CLASS, EmptyAnalyzer.class.getName(),
	analyzer+"x-terms."+AnalyzerOptions.PATTERN, "\\W+",
	analyzer+"x-splits."+AnalyzerOptions.ANALYZER_CLASS, TermCompletionAnalyzer.class.getName(),
	analyzer+"x-splits."+AnalyzerOptions.STOPWORDS, AnalyzerOptions.STOPWORDS_VALUE_NONE,
	analyzer+"x-splits."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-splits."+AnalyzerOptions.SUB_WORD_BOUNDARY, "(?<!\\p{L}|\\p{N})(?=\\p{L}|\\p{N})|(?<!\\p{Lu})(?=\\p{Lu})|(?<=\\p{N})(?=\\p{L})",
	analyzer+"x-hyphen."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "false",
	analyzer+"x-hyphen2."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen2."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen2."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen2."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "true",
	analyzer+"x-keywords."+AnalyzerOptions.ANALYZER_CLASS, KeywordAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.ANALYZER_CLASS, StandardAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.STOPWORDS, GermanAnalyzer.class.getName(),
	};
}
 
Example 2
Source Project: SourcererCC   Source File: CodeSearcher.java    License: GNU General Public License v3.0 6 votes vote down vote up
public CodeSearcher(String indexDir, String field) {
    logger.info("index directory: "+ indexDir);
    this.field = field;
    this.indexDir = indexDir;
    try {
        this.reader = DirectoryReader.open(FSDirectory.open(new File(
                this.indexDir)));
    } catch (IOException e) {
        logger.error("cant get the reader to index dir, exiting, "
                + indexDir);
        e.printStackTrace();
        System.exit(1);
    }
    this.searcher = new IndexSearcher(this.reader);
    this.analyzer = new KeywordAnalyzer();//
            //new WhitespaceAnalyzer(Version.LUCENE_46); // TODO: pass
                                                               // the
                                                               // analyzer
                                                               // as
                                                               // argument
                                                               // to
                                                               // constructor
    new CloneHelper(); // i don't remember why we are making this object?
    this.queryParser = new QueryParser(Version.LUCENE_46, this.field,
            analyzer);
}
 
Example 3
public void prepareIndex() throws IOException {
    File globalWFMDIr = new File(Util.GTPM_INDEX_DIR);
    if (!globalWFMDIr.exists()) {
        Util.createDirs(Util.GTPM_INDEX_DIR);
    }
    KeywordAnalyzer keywordAnalyzer = new KeywordAnalyzer();
    IndexWriterConfig wfmIndexWriterConfig = new IndexWriterConfig(Version.LUCENE_46, keywordAnalyzer);
    wfmIndexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
    wfmIndexWriterConfig.setRAMBufferSizeMB(1024);

    logger.info("PREPARE INDEX");
    try {
        wfmIndexWriter = new IndexWriter(FSDirectory.open(new File(Util.GTPM_INDEX_DIR)), wfmIndexWriterConfig);
        wfmIndexWriter.commit();
        wfmIndexer = new DocumentMaker(wfmIndexWriter);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 4
Source Project: SciGraph   Source File: VocabularyNeo4jImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public VocabularyNeo4jImpl(GraphDatabaseService graph,
    @Nullable @IndicatesNeo4jGraphLocation String neo4jLocation, CurieUtil curieUtil,
    NodeTransformer transformer) throws IOException {
  this.graph = graph;
  this.curieUtil = curieUtil;
  this.transformer = transformer;
  if (null != neo4jLocation) {
    Directory indexDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/node/node_auto_index"))
            .toPath());
    Directory spellDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/spellchecker"))
            .toPath());
    spellChecker = new SpellChecker(spellDirectory);
    try (IndexReader reader = DirectoryReader.open(indexDirectory)) {
      IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer());
      spellChecker.indexDictionary(new LuceneDictionary(reader, NodeProperties.LABEL
          + LuceneUtils.EXACT_SUFFIX), config, true);
    }
  } else {
    spellChecker = null;
  }
}
 
Example 5
@Before
public void setup() throws IOException {
  TableContext.clear();
  _base = new File(TMPDIR, "MutatableActionTest");
  rmr(_base);

  File file = new File(_base, TABLE);
  file.mkdirs();

  TableContext.clear();
  TableDescriptor tableDescriptor = new TableDescriptor();
  tableDescriptor.setName("test");
  tableDescriptor.setTableUri(file.toURI().toString());
  TableContext tableContext = TableContext.create(tableDescriptor);
  ShardContext shardContext = ShardContext.create(tableContext, "test");
  _action = new MutatableAction(shardContext);
  _conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
}
 
Example 6
private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
    throws IOException {
  HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
  mergePolicy.setUseCompoundFile(false);
  IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);

  Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
  int partition = partitioner.getPartition(new IntWritable(i), null, totalShardCount);
  assertEquals(i, partition);

  Document doc = getDoc(i);
  indexWriter.addDocument(doc);
  indexWriter.close();
}
 
Example 7
Source Project: incubator-retired-blur   Source File: BlurUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IndexReader getReader() throws CorruptIndexException, LockObtainFailedException, IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f1", Store.YES));

  Document doc1 = new Document();
  doc.add(new StringField("a", "b", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example 8
Source Project: incubator-retired-blur   Source File: BlurUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example 9
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example 10
@Test
public void testSymlinkWithIndexes() throws IOException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_base, "dir1"));
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer1 = new IndexWriter(dir1, conf.clone());
  writer1.addDocument(getDoc());
  writer1.close();

  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_base, "dir2"));
  IndexWriter writer2 = new IndexWriter(dir2, conf.clone());
  writer2.addIndexes(dir1);
  writer2.close();

  DirectoryReader reader1 = DirectoryReader.open(dir1);
  DirectoryReader reader2 = DirectoryReader.open(dir2);

  assertEquals(1, reader1.maxDoc());
  assertEquals(1, reader2.maxDoc());
  assertEquals(1, reader1.numDocs());
  assertEquals(1, reader2.numDocs());

  Document document1 = reader1.document(0);
  Document document2 = reader2.document(0);

  assertEquals(document1.get("id"), document2.get("id"));
}
 
Example 11
protected FieldManager newFieldManager(boolean create) throws IOException {
  return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer(), new Configuration()) {
    @Override
    protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) {
      return true;
    }

    @Override
    protected void tryToLoad(String field) {

    }

    @Override
    protected List<String> getFieldNamesToLoad() throws IOException {
      return new ArrayList<String>();
    }
  };
}
 
Example 12
Source Project: crate   Source File: LuceneOrderedDocCollectorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSearchNoScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, false);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(Float.NaN));
    assertThat(values.next().get(0), Matchers.is(Float.NaN));
}
 
Example 13
Source Project: crate   Source File: LuceneOrderedDocCollectorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSearchWithScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, true);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(1.0F));
    assertThat(values.next().get(0), Matchers.is(1.0F));
}
 
Example 14
Source Project: lucene-solr   Source File: MonitorConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
private static IndexWriterConfig defaultIndexWriterConfig() {
  IndexWriterConfig iwc = new IndexWriterConfig(new KeywordAnalyzer());
  TieredMergePolicy mergePolicy = new TieredMergePolicy();
  mergePolicy.setSegmentsPerTier(4);
  iwc.setMergePolicy(mergePolicy);
  iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
  return iwc;
}
 
Example 15
Source Project: lucene-solr   Source File: FieldTypePluginLoader.java    License: Apache License 2.0 5 votes vote down vote up
private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) {
  if (queryAnalyzer == null) return null;

  if (!(queryAnalyzer instanceof TokenizerChain)) {
    return new KeywordAnalyzer();
  }

  return ((TokenizerChain) queryAnalyzer).getMultiTermAnalyzer();
}
 
Example 16
Source Project: incubator-atlas   Source File: QueryFactory.java    License: Apache License 2.0 5 votes vote down vote up
private QueryExpression create(Request request, ResourceDefinition resourceDefinition) throws InvalidQueryException {
    String queryString;
    if (request.getCardinality() == Request.Cardinality.INSTANCE) {
        String idPropertyName = resourceDefinition.getIdPropertyName();
        queryString = String.format("%s:%s", idPropertyName, request.<String>getProperty(idPropertyName));
    } else {
        queryString = request.getQueryString();
    }

    QueryExpression queryExpression;
    if (queryString != null && !queryString.isEmpty()) {
        QueryParser queryParser = new QueryParser(Version.LUCENE_48, "name", new KeywordAnalyzer());
        queryParser.setLowercaseExpandedTerms(false);
        queryParser.setAllowLeadingWildcard(true);
        Query query;
        try {
            query = queryParser.parse((String) escape(queryString));
        } catch (ParseException e) {
            throw new InvalidQueryException(e.getMessage());
        }
        LOG.info("LuceneQuery: {}", query);
        queryExpression = create(query, resourceDefinition);
    } else {
        queryExpression = new AlwaysQueryExpression();
    }
    // add query properties to request so that they are returned
    request.addAdditionalSelectProperties(queryExpression.getProperties());
    return queryExpression;
}
 
Example 17
Source Project: querqy   Source File: LuceneQueryBuilderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
    keywordAnalyzer = new KeywordAnalyzer();
    searchFields = new HashMap<>();
    searchFields.put("f1", 1.0f);
    searchFields.put("f11", 1.0f);
    searchFields.put("f12", 1.0f);
    searchFields.put("f13", 1.0f);
    searchFields.put("f14", 1.0f);
    searchFields.put("f15", 1.0f);

    searchFields.put("f2", 2.0f);
    searchFields.put("f21", 2.0f);
    searchFields.put("f22", 2.0f);
    searchFields.put("f23", 2.0f);
    searchFields.put("f24", 2.0f);
    searchFields.put("f25", 2.0f);

    searchFields.put("f3", 3.0f);
    searchFields.put("f31", 3.0f);
    searchFields.put("f32", 3.0f);
    searchFields.put("f33", 3.0f);
    searchFields.put("f34", 3.0f);
    searchFields.put("f35", 3.0f);

    stopWords = new HashSet<>(Arrays.asList("stopA", "stopB", "stopC"));

}
 
Example 18
Source Project: querqy   Source File: DependentTermQueryBuilderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testThatResultsAreFound() throws Exception {
    ConstantFieldBoost fieldBoost = new ConstantFieldBoost(1f);
    Analyzer analyzer = new KeywordAnalyzer();

    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, analyzer);

    TestUtil.addNumDocsWithStringField("f1", "v1", indexWriter, 1);
    TestUtil.addNumDocsWithStringField("f1", "v2", indexWriter, 1);

    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader);

    DocumentFrequencyCorrection dfc = new DocumentFrequencyCorrection();

    Term term = new Term("f1", "v1");
    dfc.newClause();
    dfc.prepareTerm(term);
    dfc.finishedUserQuery();

    DependentTermQuery query = new DependentTermQuery(term, dfc, fieldBoost);

    TopDocs topDocs = indexSearcher.search(query, 10);

    assertEquals(1, topDocs.totalHits.value);
    Document resultDoc = indexSearcher.doc(topDocs.scoreDocs[0].doc);
    assertEquals("v1", resultDoc.get("f1"));

    indexReader.close();
    directory.close();
    analyzer.close();

}
 
Example 19
Source Project: jesterj   Source File: SolrSchemaUtil.java    License: Apache License 2.0 5 votes vote down vote up
private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) {
  if (queryAnalyzer == null) return null;

  if (!(queryAnalyzer instanceof TokenizerChain)) {
    return new KeywordAnalyzer();
  }
  return ((TokenizerChain) queryAnalyzer).getMultiTermAnalyzer();

}
 
Example 20
Source Project: SciGraph   Source File: VocabularyIndexAnalyzer.java    License: Apache License 2.0 5 votes vote down vote up
public VocabularyIndexAnalyzer() throws IOException, URISyntaxException {
  super(NO_REUSE_STRATEGY);
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example 21
Source Project: SciGraph   Source File: VocabularyQueryAnalyzer.java    License: Apache License 2.0 5 votes vote down vote up
public VocabularyQueryAnalyzer() {
  Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
  fieldAnalyzers.put(NodeProperties.LABEL, new TermAnalyzer());
  fieldAnalyzers.put(NodeProperties.LABEL + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.SYNONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ABREVIATION + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM, new TermAnalyzer());
  fieldAnalyzers.put(Concept.ACRONYM + LuceneUtils.EXACT_SUFFIX, new ExactAnalyzer());
  analyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers);
}
 
Example 22
private IndexReader getIndexReader() throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  Directory dir = new RAMDirectory();
  IndexWriter writer = new IndexWriter(dir, conf);
  writer.close();
  return DirectoryReader.open(dir);
}
 
Example 23
private Directory addDir(String v) throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, config);
  writer.addDocument(getDoc(v));
  writer.close();

  return directory;
}
 
Example 24
Source Project: incubator-retired-blur   Source File: CacheDirectoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test3() throws IOException, InterruptedException {
  // Thread.sleep(30000);
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(_cacheDirectory, conf);
  int docs = 100000;
  for (int i = 0; i < docs; i++) {
    if (i % 500 == 0) {
      System.out.println(i);
    }
    writer.addDocument(newDoc());
    // Thread.sleep(1);
  }
  writer.close();
  System.out.println("done writing");

  DirectoryReader reader = DirectoryReader.open(_cacheDirectory);
  System.out.println("done opening");
  assertEquals(docs, reader.numDocs());

  Document document = reader.document(0);
  System.out.println("done fetching");
  System.out.println(document);

  IndexSearcher searcher = new IndexSearcher(reader);
  TopDocs topDocs = searcher.search(new TermQuery(new Term("test", "test")), 10);
  System.out.println("done searching");
  assertEquals(docs, topDocs.totalHits);

  reader.close();
}
 
Example 25
private static Directory getDirectoryUpdateRow(String currentRowId) {
  try {
    RAMDirectory directoryUpdateRow = new RAMDirectory();
    IndexWriter writer = new IndexWriter(directoryUpdateRow, new IndexWriterConfig(Version.LUCENE_43,
        new KeywordAnalyzer()));
    Document document = new Document();
    document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
    document.add(new StringField(BlurConstants.UPDATE_ROW, currentRowId, Store.NO));
    writer.addDocument(document);
    writer.close();
    return directoryUpdateRow;
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 26
private DirectoryReader createReader() throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  Directory dir = new RAMDirectory();
  IndexWriter writer = new IndexWriter(dir, conf);
  AccessControlWriter accessControlWriter = getAccessControlFactory().getWriter();
  addDoc(writer, accessControlWriter, "r1", "d1", 0);
  addDoc(writer, accessControlWriter, "r2", "d1", 1);
  addDoc(writer, accessControlWriter, "r1", "d2", 2);
  addDoc(writer, accessControlWriter, "r2", "d2", 3);
  addDoc(writer, accessControlWriter, "r1", "d1", 4, "test");
  addDoc(writer, accessControlWriter, "r1", "d1", 5, "termmask");
  writer.close();

  return DirectoryReader.open(dir);
}
 
Example 27
Source Project: incubator-retired-blur   Source File: LoadTest.java    License: Apache License 2.0 5 votes vote down vote up
private static void createIndex(Directory directory, AccessControlFactory accessControlFactory) throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);

  AccessControlWriter accessControlWriter = accessControlFactory.getWriter();
  Random random = new Random(1);
  for (long i = 0; i < MAX_DOCS; i++) {
    if (i % 1000000 == 0) {
      System.out.println("Building " + i);
    }
    writer.addDocument(accessControlWriter.addDiscoverVisiblity("d1",
        accessControlWriter.addReadVisiblity("r1", getDoc(i, random))));
  }
  writer.close();
}
 
Example 28
@Override
protected BaseFieldManager newFieldManager(boolean create) throws IOException {
  Configuration config = new Configuration();
  Path path = new Path(DFS_FIELD_MANAGER_PATH);
  FileSystem fileSystem = path.getFileSystem(config);
  if (create) {
    fileSystem.delete(path, true);
  }
  return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config);
}
 
Example 29
private static IndexReader getReaderFlatScore(int length) throws Exception {
  _directory = new RAMDirectory();
  IndexWriter indexWriter = new IndexWriter(_directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()));
  for (int i = 0; i < length; i++) {
    Document document = new Document();
    document.add(new StringField("f1", "value", Store.NO));
    document.add(new IntDocValuesField("index", i));
    document.add(new IntField("index", i, Store.YES));
    indexWriter.addDocument(document);
  }
  indexWriter.close();
  return DirectoryReader.open(_directory);
}
 
Example 30
Source Project: incubator-retired-blur   Source File: FilterCacheTest.java    License: Apache License 2.0 5 votes vote down vote up
private void writeDocs(FilterCache filterCache, RAMDirectory directory) throws IOException {
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  conf.setCodec(new Blur024Codec());
  IndexWriter indexWriter = new IndexWriter(directory, conf);
  int count = 10000;
  addDocs(indexWriter, count);
  indexWriter.close();
}