Java Code Examples for org.apache.lucene.document.Field

The following examples show how to use org.apache.lucene.document.Field. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestIndexWriterExceptions.java    License: Apache License 2.0 6 votes vote down vote up
/** test a null bytesref value doesn't abort the entire segment */
public void testNullStoredBytesRefFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", new BytesRef("hello"));
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    BytesRef v = null;
    theField.setBytesValue(v);
    iw.addDocument(doc);
    fail("didn't get expected exception");
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 
Example 2
Source Project: lucene-solr   Source File: TestIndexReaderFunctions.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
  iwConfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
  for (String [] doc : documents) {
    Document document = new Document();
    document.add(new StringField("id", doc[0], Field.Store.NO));
    document.add(new SortedDocValuesField("id", new BytesRef(doc[0])));
    document.add(new StringField("string", doc[5], Field.Store.NO));
    document.add(new SortedDocValuesField("string", new BytesRef(doc[5])));
    document.add(new TextField("text", doc[6], Field.Store.NO));
    iw.addDocument(document);
  }

  reader = iw.getReader();
  searcher = newSearcher(reader);
  iw.close();
}
 
Example 3
Source Project: lucene-solr   Source File: TestTermVectorsWriter.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoAbortOnBadTVSettings() throws Exception {
  Directory dir = newDirectory();
  // Don't use RandomIndexWriter because we want to be sure both docs go to 1 seg:
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter iw = new IndexWriter(dir, iwc);

  Document doc = new Document();
  iw.addDocument(doc);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectors(true);
  ft.freeze();
  doc.add(new Field("field", "value", ft));

  expectThrows(IllegalArgumentException.class, () -> {
    iw.addDocument(doc);
  });

  IndexReader r = DirectoryReader.open(iw);

  // Make sure the exc didn't lose our first document:
  assertEquals(1, r.numDocs());
  iw.close();
  r.close();
  dir.close();
}
 
Example 4
Source Project: Quelea   Source File: SongSearchIndex.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Add a number of songs to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param songList the song list to add.
 */
@Override
public synchronized void addAll(Collection<? extends SongDisplayable> songList) {
    Pattern p = Pattern.compile("[^\\w\\s]", Pattern.UNICODE_CHARACTER_CLASS);
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for (SongDisplayable song : songList) {
            Document doc = new Document();
            if (song.getTitle() != null) {
                doc.add(new TextField("title", p.matcher(song.getTitle()).replaceAll(""), Field.Store.NO));
            }
            if (song.getAuthor() != null) {
                doc.add(new TextField("author", p.matcher(song.getAuthor()).replaceAll(""), Field.Store.NO));
            }
            if (song.getLyrics(false, false, false) != null) {
                doc.add(new TextField("lyrics", p.matcher(song.getLyrics(false, false, false)).replaceAll(""), Field.Store.NO));
            }
            doc.add(new TextField("number", p.matcher(Long.toString(song.getID())).replaceAll(""), Field.Store.YES));
            writer.addDocument(doc);
            songs.put(song.getID(), song);
            LOGGER.log(Level.FINE, "Added song to index: {0}", song.getTitle());
        }
    } catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 
Example 5
Source Project: SnowGraph   Source File: CodePatternSearcher.java    License: Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example 6
Source Project: lucene-solr   Source File: TestTransactionRollback.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();

  //Build index, of records 1 to 100, committing after each batch of 10
  IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
  IndexWriter w=new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                        .setIndexDeletionPolicy(sdp));

  for(int currentRecordId=1;currentRecordId<=100;currentRecordId++) {
    Document doc=new Document();
    doc.add(newTextField(FIELD_RECORD_ID, ""+currentRecordId, Field.Store.YES));
    w.addDocument(doc);

    if (currentRecordId%10 == 0) {
      Map<String,String> data = new HashMap<>();
      data.put("index", "records 1-"+currentRecordId);
      w.setLiveCommitData(data.entrySet());
      w.commit();
    }
  }

  w.close();
}
 
Example 7
Source Project: dependency-track   Source File: LicenseIndexer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds a License object to a Lucene index.
 *
 * @param license A persisted License object.
 */
public void add(final License license) {
    final Document doc = new Document();
    addField(doc, IndexConstants.LICENSE_UUID, license.getUuid().toString(), Field.Store.YES, false);
    addField(doc, IndexConstants.LICENSE_LICENSEID, license.getLicenseId(), Field.Store.YES, true);
    addField(doc, IndexConstants.LICENSE_NAME, license.getName(), Field.Store.YES, true);

    try {
        getIndexWriter().addDocument(doc);
    } catch (IOException e) {
        LOGGER.error("An error occurred while adding a license to the index", e);
        Notification.dispatch(new Notification()
                .scope(NotificationScope.SYSTEM)
                .group(NotificationGroup.INDEXING_SERVICE)
                .title(NotificationConstants.Title.LICENSE_INDEXER)
                .content("An error occurred while adding a license to the index. Check log for details. " + e.getMessage())
                .level(NotificationLevel.ERROR)
        );
    }
}
 
Example 8
Source Project: SnowGraph   Source File: InMemoryIndex.java    License: Apache License 2.0 6 votes vote down vote up
public InMemoryIndex(Map<String,String> id2Text){
    Analyzer analyzer = new EnglishAnalyzer();
    IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    try {
        IndexWriter writer = new IndexWriter(directory, iwc);
        for (String id:id2Text.keySet()) {
            Document doc=new Document();
            doc.add(new StringField("id", id, Field.Store.YES));
            doc.add(new TextField("content", id2Text.get(id), Field.Store.YES));
            writer.addDocument(doc);
        }
        writer.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 9
Source Project: lucene-solr   Source File: TestConstantScoreQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testPropagatesApproximations() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  Field f = newTextField("field", "a b", Field.Store.NO);
  doc.add(f);
  w.addDocument(doc);
  w.commit();

  DirectoryReader reader = w.getReader();
  final IndexSearcher searcher = newSearcher(reader);
  searcher.setQueryCache(null); // to still have approximations

  PhraseQuery pq = new PhraseQuery("field", "a", "b");

  Query q = searcher.rewrite(new ConstantScoreQuery(pq));

  final Weight weight = searcher.createWeight(q, ScoreMode.COMPLETE, 1);
  final Scorer scorer = weight.scorer(searcher.getIndexReader().leaves().get(0));
  assertNotNull(scorer.twoPhaseIterator());

  reader.close();
  w.close();
  dir.close();
}
 
Example 10
Source Project: lucene-solr   Source File: TestTermVectorsReader.java    License: Apache License 2.0 6 votes vote down vote up
public void testIllegalVectorOffsetsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorOffsets(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector offsets for a field that is not indexed (field=\"field\")", expected.getMessage());
  
  w.close();
  dir.close();
}
 
Example 11
Source Project: lucene-solr   Source File: TestIndexWriter.java    License: Apache License 2.0 6 votes vote down vote up
public void testNRTReaderVersion() throws Exception {
  Directory d = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
  IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  doc.add(newStringField("id", "0", Field.Store.YES));
  w.addDocument(doc);
  DirectoryReader r = w.getReader();
  long version = r.getVersion();
  r.close();

  w.addDocument(doc);
  r = w.getReader();
  long version2 = r.getVersion();
  r.close();
  assert(version2 > version);

  w.deleteDocuments(new Term("id", "0"));
  r = w.getReader();
  w.close();
  long version3 = r.getVersion();
  r.close();
  assert(version3 > version2);
  d.close();
}
 
Example 12
Source Project: lucene-solr   Source File: TestSimilarity2.java    License: Apache License 2.0 6 votes vote down vote up
/** similar to the above, however the field exists, but we query with a term that doesnt exist too */
public void testEmptyTerm() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("foo", "bar", Field.Store.NO));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);
  
  for (Similarity sim : sims) {
    is.setSimilarity(sim);
    BooleanQuery.Builder query = new BooleanQuery.Builder();
    query.add(new TermQuery(new Term("foo", "bar")), BooleanClause.Occur.SHOULD);
    query.add(new TermQuery(new Term("foo", "baz")), BooleanClause.Occur.SHOULD);
    assertEquals(1, is.search(query.build(), 10).totalHits.value);
  }
  ir.close();
  dir.close();
}
 
Example 13
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testExcIndexingDocBeforeDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setDocValuesType(DocValuesType.SORTED);
  ft.freeze();
  Field field = new Field("test", "value", ft);
  field.setTokenStream(new TokenStream() {
      @Override
      public boolean incrementToken() {
        throw new RuntimeException("no");
      }
    });
  doc.add(field);
  expectThrows(RuntimeException.class, () -> {
    w.addDocument(doc);
  });

  w.addDocument(new Document());
  w.close();
  dir.close();
}
 
Example 14
Source Project: lucene-solr   Source File: TestPostingsOffsets.java    License: Apache License 2.0 6 votes vote down vote up
private void checkTokens(Token[] field1, Token[] field2) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);
   
    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(field1), ft));
    doc.add(new Field("body", new CannedTokenStream(field2), ft));
    riw.addDocument(doc);
    riw.close();
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}
 
Example 15
Source Project: crate   Source File: EngineTestCase.java    License: Apache License 2.0 6 votes vote down vote up
protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate,
    boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}
 
Example 16
Source Project: lucene-solr   Source File: TestDirectSpellChecker.java    License: Apache License 2.0 6 votes vote down vote up
public void testBogusField() throws Exception {
  DirectSpellChecker spellChecker = new DirectSpellChecker();
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer);

  for (int i = 0; i < 20; i++) {
    Document doc = new Document();
    doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO));
    writer.addDocument(doc);
  }

  IndexReader ir = writer.getReader();

  SuggestWord[] similar = spellChecker.suggestSimilar(new Term(
      "bogusFieldBogusField", "fvie"), 2, ir,
      SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
  assertEquals(0, similar.length);
  
  IOUtils.close(ir, writer, dir, analyzer);
}
 
Example 17
Source Project: lucene-solr   Source File: TestLongValuesSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = TestUtil.nextInt(random(), 2049, 4000);
  int leastValue = 45;
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    if (i == 545)
      document.add(new NumericDocValuesField("onefield", LEAST_LONG_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example 18
public void testBasics() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Field body = new Field("body", "", fieldType);
  Document doc = new Document();
  doc.add(body);

  body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
  iw.addDocument(doc);
  body.setStringValue("Highlighting the first term. Hope it works.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  Query query = new IntervalQuery("body", Intervals.term("highlighting"));
  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  assertEquals(2, topDocs.totalHits.value);
  String snippets[] = highlighter.highlight("body", query, topDocs);
  assertEquals(2, snippets.length);
  assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
  assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
  ir.close();
}
 
Example 19
Source Project: lucene-solr   Source File: TestIndexWriterExceptions.java    License: Apache License 2.0 6 votes vote down vote up
/** test a null string value doesn't abort the entire segment */
public void testNullStoredFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", "hello", StoredField.TYPE);
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    theField.setStringValue(null);
    iw.addDocument(doc);
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 
Example 20
/**
 * Make sure highlighter returns first N sentences if
 * there were no hits.
 */
public void testEmptyHighlights() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Document doc = new Document();

  Field body = new Field("body", "test this is.  another sentence this test has.  far away is that planet.", fieldType);
  doc.add(body);
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  Query query = new IntervalQuery("body", Intervals.term("highlighting"));
  int[] docIDs = new int[] {0};
  String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");
  assertEquals(1, snippets.length);
  assertEquals("test this is.  another sentence this test has.  ", snippets[0]);
  ir.close();
}
 
Example 21
Source Project: lucene-solr   Source File: TestExpressionSorts.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(500);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example 22
Source Project: lucene-solr   Source File: TestFieldCacheSort.java    License: Apache License 2.0 6 votes vote down vote up
/** test that we throw exception on multi-valued field, creates corrupt reader, use SORTED_SET instead */
public void testMultiValuedField() throws IOException {
  Directory indexStore = newDirectory();
  IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(new MockAnalyzer(random())));
  for(int i=0; i<5; i++) {
      Document doc = new Document();
      doc.add(new StringField("string", "a"+i, Field.Store.NO));
      doc.add(new StringField("string", "b"+i, Field.Store.NO));
      writer.addDocument(doc);
  }
  writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
  writer.close();
  Sort sort = new Sort(
      new SortField("string", SortField.Type.STRING),
      SortField.FIELD_DOC);
  IndexReader reader = UninvertingReader.wrap(DirectoryReader.open(indexStore),
                       Collections.singletonMap("string", Type.SORTED));
  IndexSearcher searcher = new IndexSearcher(reader);
  expectThrows(IllegalStateException.class, () -> {
    searcher.search(new MatchAllDocsQuery(), 500, sort);
  });
  reader.close();
  indexStore.close();
}
 
Example 23
Source Project: Elasticsearch   Source File: TokenCountFieldMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = StringFieldMapper.parseCreateFieldForString(context, null /* Out null value is an int so we convert*/, fieldType().boost());
    if (valueAndBoost.value() == null && fieldType().nullValue() == null) {
        return;
    }

    if (fieldType().indexOptions() != NONE || fieldType().stored() || fieldType().hasDocValues()) {
        int count;
        if (valueAndBoost.value() == null) {
            count = fieldType().nullValue();
        } else {
            count = countPositions(analyzer, simpleName(), valueAndBoost.value());
        }
        addIntegerFields(context, fields, count, valueAndBoost.boost());
    }
}
 
Example 24
Source Project: lucene-solr   Source File: CoreParserTestIndexData.java    License: Apache License 2.0 6 votes vote down vote up
CoreParserTestIndexData(Analyzer analyzer) throws Exception {
  BufferedReader d = new BufferedReader(new InputStreamReader(
      TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
  dir = LuceneTestCase.newDirectory();
  IndexWriter writer = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(analyzer));
  String line = d.readLine();
  while (line != null) {
    int endOfDate = line.indexOf('\t');
    String date = line.substring(0, endOfDate).trim();
    String content = line.substring(endOfDate).trim();
    Document doc = new Document();
    doc.add(LuceneTestCase.newTextField("date", date, Field.Store.YES));
    doc.add(LuceneTestCase.newTextField("contents", content, Field.Store.YES));
    doc.add(new IntPoint("date3", Integer.parseInt(date)));
    writer.addDocument(doc);
    line = d.readLine();
  }
  d.close();
  writer.close();
  reader = DirectoryReader.open(dir);
  searcher = LuceneTestCase.newSearcher(reader, false);
}
 
Example 25
Source Project: lucene-solr   Source File: TestPerFieldPostingsFormat2.java    License: Apache License 2.0 5 votes vote down vote up
private void addDocs2(IndexWriter writer, int numDocs) throws IOException {
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(newTextField("content", "bbb", Field.Store.NO));
    writer.addDocument(doc);
  }
}
 
Example 26
Source Project: lucene-solr   Source File: BaseExplanationTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public static Document createDoc(int index) {
  Document doc = new Document();
  doc.add(newStringField(KEY, ""+index, Field.Store.NO));
  doc.add(new SortedDocValuesField(KEY, new BytesRef(""+index)));
  Field f = newTextField(FIELD, docFields[index], Field.Store.NO);
  doc.add(f);
  doc.add(newTextField(ALTFIELD, docFields[index], Field.Store.NO));
  return doc;
}
 
Example 27
public void testMaxLengthWithMultivalue() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Document doc = new Document();

  final String value = "This is a multivalued field. Sentencetwo field.";
  doc.add(new Field("body", value, fieldType));
  doc.add(new Field("body", value, fieldType));
  doc.add(new Field("body", value, fieldType));

  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  highlighter.setMaxLength(value.length() * 2 + 1);
  Query query = new IntervalQuery("body", Intervals.term("field"));
  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  assertEquals(1, topDocs.totalHits.value);
  String snippets[] = highlighter.highlight("body", query, topDocs, 10);
  assertEquals(1, snippets.length);
  String highlightedValue = "This is a multivalued <b>field</b>. Sentencetwo <b>field</b>.";
  assertEquals(highlightedValue + "... " + highlightedValue, snippets[0]);
  ir.close();
}
 
Example 28
Source Project: lucene-solr   Source File: TestCustomTermFreq.java    License: Apache License 2.0 5 votes vote down vote up
public void testInvalidProx() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));

  Document doc = new Document();
  FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
  Field field = new Field("field",
                          new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"},
                                              new int[] {42, 128, 17, 100}),
                          fieldType);
  doc.add(field);
  Exception e = expectThrows(IllegalStateException.class, () -> {w.addDocument(doc);});
  assertEquals("field \"field\": cannot index positions while using custom TermFrequencyAttribute", e.getMessage());
  IOUtils.close(w, dir);
}
 
Example 29
Source Project: lucene-solr   Source File: TestSimpleMatcher.java    License: Apache License 2.0 5 votes vote down vote up
public void testSimpleMatcher() throws IOException {

    try (Monitor monitor = newMonitor()) {
      monitor.register(
          new MonitorQuery("1", parse("test")),
          new MonitorQuery("2", parse("wibble")));
      Document doc = new Document();
      doc.add(newTextField(FIELD, "test", Field.Store.NO));

      MatchingQueries<QueryMatch> matches = monitor.match(doc, QueryMatch.SIMPLE_MATCHER);
      assertNotNull(matches.matches("1"));
    }
  }
 
Example 30
Source Project: lucene-solr   Source File: TestTermAutomatonQuery.java    License: Apache License 2.0 5 votes vote down vote up
public void testRewriteNoMatch() throws Exception {
  TermAutomatonQuery q = new TermAutomatonQuery("field");
  int initState = q.createState();
  q.finish();
  
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("field", "x y z", Field.Store.NO));
  w.addDocument(doc);

  IndexReader r = w.getReader();
  assertTrue(q.rewrite(r) instanceof MatchNoDocsQuery);
  IOUtils.close(w, r, dir);
}