org.apache.lucene.analysis.standard.StandardAnalyzer Java Examples

public static void main(String[] args) throws Exception {
    // index
    try (Directory index = new NIOFSDirectory(Paths.get("/tmp/index"))) {
        // add
        try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer()))) {
            Document doc = new Document();
            doc.add(new TextField("blog", "yuzhouwan.com", Field.Store.YES));
            doc.add(new StringField("github", "asdf2014", Field.Store.YES));
            writer.addDocument(doc);
            writer.commit();
        }
        // search
        try (DirectoryReader reader = DirectoryReader.open(index)) {
            IndexSearcher searcher = new IndexSearcher(reader);
            QueryParser parser = new QueryParser("blog", new StandardAnalyzer());
            Query query = parser.parse("yuzhouwan.com");
            ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
            for (ScoreDoc hit : hits) {
                Document hitDoc = searcher.doc(hit.doc);
                System.out.println(hitDoc.get("blog"));
            }
        }
    }
}
 

@Test
public void shouldAddTermsFromSortOrderAsc() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("order")).thenReturn("asc");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams().asc()), getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

/**
 * Search for the user-specified query expression in the current page.
 * @throws Exception if an error occurs.
 */
private void search() throws Exception {
  final QueryParser parser = new QueryParser("contents", new StandardAnalyzer());
  final Query q = parser.parse(query);

  final MemoryIndex index = new MemoryIndex();
  final Link link = new Link(url);
  final PageData pageData = new SimpleHttpClientParser().load(link);
  index.addField("contents", pageData.getData().toString(), new StandardAnalyzer());
  final IndexSearcher searcher = index.createSearcher();
  final Hits hits = searcher.search(q);
  @SuppressWarnings("rawtypes")
  final Iterator it = hits.iterator();
  float relevance = 0f;
  if (it.hasNext()) {
    while (it.hasNext()) {
      final Hit hit = (Hit) it.next();
      relevance += ((float) Math.round(hit.getScore() * 1000)) / 10;
    }
    matchedLinks.add(new LinkMatch(url, relevance));
  }
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfSunion() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sunion");
  when(localParamsMock.get("key")).thenReturn("key1");
  when(localParamsMock.get("key1")).thenReturn("key2");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "key1").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sunion(anyString(), anyString())).thenReturn(new HashSet<String>());
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).sunion("key1", "key2");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Test
public void shouldDeflateGzipAndParseJsonTermsFromRedisOnGetCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("get");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("compression")).thenReturn("gzip");
  when(localParamsMock.get("serialization")).thenReturn("json");

  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.get(any(byte[].class))).thenReturn(Compressor.compressGzip("[100,200,300]".getBytes()));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).get("simpleKey".getBytes());
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(3, terms.size());
}
 

public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
	String line = value.toString();

	CharArraySet noise = EnglishAnalyzer.getDefaultStopSet();
	// We are using a standard tokenizer that eliminates the stop words.
	// We can use Stemming tokenizer such Porter
	// A set of English noise keywords is used that will eliminates
	// words such as "the, a, etc"
	Analyzer analyzer = new StandardAnalyzer(noise);
	List<String> token = Tokenizer.tokenizeString(analyzer, line);
	Iterator<String> it = token.iterator();
	while (it.hasNext()) {
		word.set(it.next());
		fileName.set(key);
		if (!mapTable.containsKey(fileName.toString() + word.toString())) {
			context.write(fileName, word);
			mapTable.put(fileName.toString() + word.toString(), new IntWritable(1));
		}
	}
}
 

@Override
String[] getExtraProperties() {
	String analyzer = ConfigurableAnalyzerFactory.Options.ANALYZER;
	return new String[]{
	FullTextIndex.Options.ANALYZER_FACTORY_CLASS, ConfigurableAnalyzerFactory.class.getName(),
	analyzer+"_."+AnalyzerOptions.LIKE, "x-empty",
	analyzer+"x-empty."+AnalyzerOptions.ANALYZER_CLASS, EmptyAnalyzer.class.getName(),
	analyzer+"x-terms."+AnalyzerOptions.PATTERN, "\\W+",
	analyzer+"x-splits."+AnalyzerOptions.ANALYZER_CLASS, TermCompletionAnalyzer.class.getName(),
	analyzer+"x-splits."+AnalyzerOptions.STOPWORDS, AnalyzerOptions.STOPWORDS_VALUE_NONE,
	analyzer+"x-splits."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-splits."+AnalyzerOptions.SUB_WORD_BOUNDARY, "(?<!\\p{L}|\\p{N})(?=\\p{L}|\\p{N})|(?<!\\p{Lu})(?=\\p{Lu})|(?<=\\p{N})(?=\\p{L})",
	analyzer+"x-hyphen."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "false",
	analyzer+"x-hyphen2."+AnalyzerOptions.SUB_WORD_BOUNDARY, "[-.]",
	analyzer+"x-hyphen2."+AnalyzerOptions.SOFT_HYPHENS, "-",
	analyzer+"x-hyphen2."+AnalyzerOptions.WORD_BOUNDARY, " ",
	analyzer+"x-hyphen2."+AnalyzerOptions.ALWAYS_REMOVE_SOFT_HYPHENS, "true",
	analyzer+"x-keywords."+AnalyzerOptions.ANALYZER_CLASS, KeywordAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.ANALYZER_CLASS, StandardAnalyzer.class.getName(),
	analyzer+"en-x-de."+AnalyzerOptions.STOPWORDS, GermanAnalyzer.class.getName(),
	};
}
 

@Test
public void shouldAddTermsFromRedisOnLrangeCommandCustomMinAndMax() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("lrange");
  when(localParamsMock.get("min")).thenReturn("2");
  when(localParamsMock.get("max")).thenReturn("3");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.lrange(anyString(), anyLong(), anyLong())).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).lrange("simpleKey", 2, 3);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Inject
public LucenePerUserWaveViewHandlerImpl(IndexDirectory directory,
                                        ReadableWaveletDataProvider waveletProvider,
                                        @Named(CoreSettingsNames.WAVE_SERVER_DOMAIN) String domain,
                                        @IndexExecutor Executor executor) {
  this.waveletProvider = waveletProvider;
  this.executor = executor;
  analyzer = new StandardAnalyzer(LUCENE_VERSION);
  try {
    IndexWriterConfig indexConfig = new IndexWriterConfig(LUCENE_VERSION, analyzer);
    indexConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
    indexWriter = new IndexWriter(directory.getDirectory(), indexConfig);
    nrtManager = new NRTManager(indexWriter, new WaveSearchWarmer(domain));
  } catch (IOException ex) {
    throw new IndexException(ex);
  }

  nrtManagerReopenThread = new NRTManagerReopenThread(nrtManager, MAX_STALE_SEC, MIN_STALE_SEC);
  nrtManagerReopenThread.start();
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfSinter() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sinter");
  when(localParamsMock.get("key")).thenReturn("key1");
  when(localParamsMock.get("key1")).thenReturn("key2");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "key1").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sinter(anyString(), anyString())).thenReturn(new HashSet<String>());
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).sinter("key1", "key2");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Transactional(readOnly = true)
public Integer createIndex(Integer siteId, Integer channelId,
		Date startDate, Date endDate, Integer startId, Integer max,
		Directory dir) throws IOException, ParseException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		if (exist) {
			LuceneContent.delete(siteId, channelId, startDate, endDate,
					writer);
		}
		Integer lastId = luceneContentDao.index(writer, siteId, channelId,
				startDate, endDate, startId, max);
		writer.optimize();
		return lastId;
	} finally {
		writer.close();
	}
}
 

@Test
public void shouldAddTermsFromRedisOnZrangeCommandWithCustomRange() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("zrange");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("range_start")).thenReturn("1");
  when(localParamsMock.get("range_end")).thenReturn("100");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.zrangeWithScores(anyString(), anyLong(), anyLong()))
      .thenReturn(new HashSet<>(Arrays.asList(new Tuple("123", (double) 1.0f), new Tuple("321", (double) 1.0f))));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).zrangeWithScores("simpleKey", 1, 100);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Override
public boolean initialize(ResourceSpecifier aSpecifier, Map<String, Object> aAdditionalParams)
        throws ResourceInitializationException {
  super.initialize(aSpecifier, aAdditionalParams);
  hits = Integer.class.cast(getParameterValue("hits"));
  // query constructor
  String stoplistPath = String.class.cast(getParameterValue("stoplist-path"));
  try {
    stoplist = Resources.readLines(getClass().getResource(stoplistPath), UTF_8).stream()
            .map(String::trim).collect(toSet());
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
  analyzer = new StandardAnalyzer();
  parser = new QueryParser("text", analyzer);
  return true;
}
 

@Test
public void shouldAddTermsFromRedisOnLrangeCommandEmptyMinAndMaxFallsBackToDefault() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("lrange");
  when(localParamsMock.get("min")).thenReturn("");
  when(localParamsMock.get("max")).thenReturn("");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.lrange(anyString(), anyLong(), anyLong())).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).lrange("simpleKey", 0, -1);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 100);
  analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  queryStringConstructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          LuceneQueryStringConstructor.class, QueryStringConstructor.class);
  parser = new QueryParser("text", analyzer);
  // load parameters
  String param = UimaContextHelper.getConfigParameterStringValue(context, "doc-logreg-params");
  try {
    docFeatWeights = Resources.readLines(getClass().getResource(param), UTF_8).stream().limit(1)
            .map(line -> line.split("\t")).flatMap(Arrays::stream)
            .mapToDouble(Double::parseDouble).toArray();
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfHget() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("hget");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("field")).thenReturn("f1");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.hget(anyString(), anyString())).thenReturn(null);
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).hget("simpleKey", "f1");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Test
public void shouldAddTermsFromSort() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams()), getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

private void doMLTSearch() {
  if (Objects.isNull(mltDocFTF.getValue())) {
    throw new LukeException("Doc num is not set.");
  }
  int docNum = (int) mltDocFTF.getValue();
  MLTConfig mltConfig = operatorRegistry.get(MLTTabOperator.class)
      .map(MLTTabOperator::getConfig)
      .orElse(new MLTConfig.Builder().build());
  Analyzer analyzer = operatorRegistry.get(AnalysisTabOperator.class)
      .map(AnalysisTabOperator::getCurrentAnalyzer)
      .orElse(new StandardAnalyzer());
  Query query = searchModel.mltQuery(docNum, mltConfig, analyzer);
  Set<String> fieldsToLoad = operatorRegistry.get(FieldValuesTabOperator.class)
      .map(FieldValuesTabOperator::getFieldsToLoad)
      .orElse(Collections.emptySet());
  SearchResults results = searchModel.search(query, new SimilarityConfig.Builder().build(), fieldsToLoad, DEFAULT_PAGE_SIZE, false);

  TableUtils.setupTable(resultsTable, ListSelectionModel.SINGLE_SELECTION, new SearchResultsTableModel(), null,
      SearchResultsTableModel.Column.DOCID.getColumnWidth(),
      SearchResultsTableModel.Column.SCORE.getColumnWidth());
  populateResults(results);

  messageBroker.clearStatusMessage();
}
 

@Test
public void shouldAddTermsFromRedisOnHmgetCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("hmget");
  when(localParamsMock.get("key")).thenReturn("hash");
  when(localParamsMock.get("field")).thenReturn("field1");
  when(localParamsMock.getParameterNamesIterator()).thenReturn(Arrays.asList("command", "key", "field").iterator());
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.hmget(anyString(), anyString())).thenReturn(Arrays.asList("123"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).hmget("hash", "field1");
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(1, terms.size());
}
 

@Test
public void shouldAddTermsFromSortOffset() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("sort");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get("offset")).thenReturn("100");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.sort(anyString(), any(SortingParams.class))).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  final ArgumentCaptor<SortingParams> argument = ArgumentCaptor.forClass(SortingParams.class);
  verify(jedisMock).sort(eq("simpleKey"), argument.capture());
  Assert.assertEquals(getSortingParamString(new SortingParams().limit(100, 0)),
      getSortingParamString(argument.getValue()));
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

public void testFreeFormQueryParse() throws Exception {
    String queryString = new String("name:spell -description:another");
    log.info("Original query: "  + queryString);

    NGramQueryParser parser = new NGramQueryParser("name",
            new NGramAnalyzer(min_ngram, max_ngram), true);
    Query q = parser.parse(queryString);
    log.info("NGramQueryParser parsed query:  " + q.toString());

    QueryParser origParser = new QueryParser("name", new StandardAnalyzer());
    q = origParser.parse(queryString);
    log.info("QueryParser parsed query = " + q.toString());
}
 

@Test
public void shouldReturnEmptyQueryOnEmptyListOfSrandmember() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("srandmember");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.srandmember(anyString(), anyInt())).thenReturn(new ArrayList<String>());
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).srandmember("simpleKey", 1);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(0, terms.size());
}
 

@Transactional(readOnly = true)
public void deleteIndex(Integer contentId, Directory dir)
		throws IOException, ParseException {
	boolean exist = IndexReader.indexExists(dir);
	if (exist) {
		IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
				Version.LUCENE_30), false,
				IndexWriter.MaxFieldLength.LIMITED);
		try {
			LuceneContent.delete(contentId, writer);
		} finally {
			writer.close();
		}
	}
}
 

/**
 * @param args
 */
public static void main(String[] args) throws Exception{
  if (args.length != 2) {
    System.out.println("usage: source_file index_dir");
    System.exit(1);
  }
  File f = new File(args[0]);
  BufferedReader reader = new BufferedReader(new FileReader(f));

  ObjectMapper mapper = new ObjectMapper();

  IndexWriterConfig idxWriterConfig = new IndexWriterConfig(new StandardAnalyzer());
  Directory dir = FSDirectory.open(FileSystems.getDefault().getPath(args[1]));
  IndexWriter writer = new IndexWriter(dir, idxWriterConfig);
  int count = 0;
  while (true) {
    String line = reader.readLine();
    if (line == null) break;
    JsonNode json = mapper.readTree(line);
    Document doc = buildDoc(json);
    writer.addDocument(doc);
    count++;
    if (count % 100 == 0) {
      System.out.print(".");
    }
  }
  
  System.out.println(count+" docs indexed");
  
  reader.close();
  writer.commit();
  writer.close();
}
 

@Test
public void shouldAddTermsFromRedisOnLrangeCommand() throws SyntaxError, IOException {
  when(localParamsMock.get("command")).thenReturn("lrange");
  when(localParamsMock.get("key")).thenReturn("simpleKey");
  when(localParamsMock.get(QueryParsing.V)).thenReturn("string_field");
  when(jedisMock.lrange(anyString(), anyLong(), anyLong())).thenReturn(Arrays.asList("123", "321"));
  when(requestMock.getSchema()).thenReturn(schema);
  when(schema.getQueryAnalyzer()).thenReturn(new StandardAnalyzer());
  redisQParser = new RedisQParser("string_field", localParamsMock, paramsMock, requestMock, commandHandler);
  final Query query = redisQParser.parse();
  verify(jedisMock).lrange("simpleKey", 0, -1);
  IndexSearcher searcher = new IndexSearcher(new MultiReader());
  final Set<Term> terms = extractTerms(searcher, query);
  Assert.assertEquals(2, terms.size());
}
 

public HashMap<String,Short> indexFile(int i, String filePath) {
    HashMap<String,Short> map = new HashMap<>();
    try {
        List<String> lines = FileUtils.readLines(new File(filePath));
        Analyzer analyzer = new StandardAnalyzer();
        int messageCountFile = 0;
        for(String line : lines){
            if(!mention || (mention && line.contains("@"))){
                messageCountFile++;
                String cleanLine = line.toLowerCase();
                List<String> strings = Tokenizer.tokenizeString(analyzer, cleanLine);
                for(String string : strings){
                    if(string.length()>=minWordLength){
                        Short count = map.get(string);
                        if(count == null){
                            count = 0;
                        }
                        count++;
                        map.put(string,count);
                    }
                }
            }
        }
        messageCountDistribution.put(i,messageCountFile);
        messageCount += messageCountFile;
    } catch (IOException ex) {
        Logger.getLogger(Indexer.class.getName()).log(Level.SEVERE, null, ex);
    }
    return map;
}
 

@Test
public void givenPrefixQueryWhenFetchedDocumentThenCorrect() {
    InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
    inMemoryLuceneIndex.indexDocument("article", "Lucene introduction");
    inMemoryLuceneIndex.indexDocument("article", "Introduction to Lucene");

    Term term = new Term("body", "intro");
    Query query = new PrefixQuery(term);

    List<Document> documents = inMemoryLuceneIndex.searchIndex(query);
    Assert.assertEquals(2, documents.size());
}
 

public IndexSearcher() {
    try {
        searcher = new org.apache.lucene.search.IndexSearcher(new ClasspathDirectory());
    } catch (IOException e) {
        e.printStackTrace();
    }
    analyzer = new StandardAnalyzer(Version.LUCENE_31);
    parser = new MultiFieldQueryParser(Version.LUCENE_31, new String[]{"name","description"}, analyzer);
}
 

public void testDemo() throws IOException {
  String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
  String text = "This is the text to be indexed. " + longTerm;

  Path indexPath = Files.createTempDirectory("tempIndex");
  try (Directory dir = FSDirectory.open(indexPath)) {
    Analyzer analyzer = new StandardAnalyzer();
    try (IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer))) {
      Document doc = new Document();
      doc.add(newTextField("fieldname", text, Field.Store.YES));
      iw.addDocument(doc);
    }

    // Now search the index.
    try (IndexReader reader = DirectoryReader.open(dir)) {
      IndexSearcher searcher = newSearcher(reader);

      assertEquals(1, searcher.count(new TermQuery(new Term("fieldname", longTerm))));

      Query query = new TermQuery(new Term("fieldname", "text"));
      TopDocs hits = searcher.search(query, 1);
      assertEquals(1, hits.totalHits.value);

      // Iterate through the results.
      for (int i = 0; i < hits.scoreDocs.length; i++) {
        Document hitDoc = searcher.doc(hits.scoreDocs[i].doc);
        assertEquals(text, hitDoc.get("fieldname"));
      }

      // Test simple phrase query.
      PhraseQuery phraseQuery = new PhraseQuery("fieldname", "to", "be");
      assertEquals(1, searcher.count(phraseQuery));
    }
  }

  IOUtils.rm(indexPath);
}
 

private void indexWithLucene() throws IOException {
    System.out.println("Deleting old Lucene index");
    FileUtils.deleteDirectory(new File(LUCENE_INDEX));
    System.out.println("Indexing with Lucene");
    final BufferedReader dictionary = config.newReader();
    try {
        final Directory directory = FSDirectory.open(new File(LUCENE_INDEX));
        try {
            final IndexWriter luceneWriter = new IndexWriter(directory,
                    new StandardAnalyzer(LuceneSearch.LUCENE_VERSION), true,
                    IndexWriter.MaxFieldLength.UNLIMITED);
            try {
                final IDictParser parser = config.fileType.newParser(config);
                indexWithLucene(dictionary, luceneWriter, parser);
                System.out.println("Optimizing Lucene index");
                luceneWriter.optimize();
            } finally {
                luceneWriter.close();
            }
        } finally {
            closeQuietly(directory);
        }
    } finally {
        IOUtils.closeQuietly(dictionary);
    }
    System.out.println("Finished Lucene indexing");
}