org.apache.lucene.document.Field Java Examples

The following examples show how to use org.apache.lucene.document.Field. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestIndexReaderFunctions.java    From lucene-solr with Apache License 2.0 7 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
  iwConfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
  for (String [] doc : documents) {
    Document document = new Document();
    document.add(new StringField("id", doc[0], Field.Store.NO));
    document.add(new SortedDocValuesField("id", new BytesRef(doc[0])));
    document.add(new StringField("string", doc[5], Field.Store.NO));
    document.add(new SortedDocValuesField("string", new BytesRef(doc[5])));
    document.add(new TextField("text", doc[6], Field.Store.NO));
    iw.addDocument(document);
  }

  reader = iw.getReader();
  searcher = newSearcher(reader);
  iw.close();
}
 
Example #2
Source File: CoreParserTestIndexData.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
CoreParserTestIndexData(Analyzer analyzer) throws Exception {
  BufferedReader d = new BufferedReader(new InputStreamReader(
      TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
  dir = LuceneTestCase.newDirectory();
  IndexWriter writer = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(analyzer));
  String line = d.readLine();
  while (line != null) {
    int endOfDate = line.indexOf('\t');
    String date = line.substring(0, endOfDate).trim();
    String content = line.substring(endOfDate).trim();
    Document doc = new Document();
    doc.add(LuceneTestCase.newTextField("date", date, Field.Store.YES));
    doc.add(LuceneTestCase.newTextField("contents", content, Field.Store.YES));
    doc.add(new IntPoint("date3", Integer.parseInt(date)));
    writer.addDocument(doc);
    line = d.readLine();
  }
  d.close();
  writer.close();
  reader = DirectoryReader.open(dir);
  searcher = LuceneTestCase.newSearcher(reader, false);
}
 
Example #3
Source File: CodePatternSearcher.java    From SnowGraph with Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example #4
Source File: SongSearchIndex.java    From Quelea with GNU General Public License v3.0 6 votes vote down vote up
/**
 * Add a number of songs to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param songList the song list to add.
 */
@Override
public synchronized void addAll(Collection<? extends SongDisplayable> songList) {
    Pattern p = Pattern.compile("[^\\w\\s]", Pattern.UNICODE_CHARACTER_CLASS);
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for (SongDisplayable song : songList) {
            Document doc = new Document();
            if (song.getTitle() != null) {
                doc.add(new TextField("title", p.matcher(song.getTitle()).replaceAll(""), Field.Store.NO));
            }
            if (song.getAuthor() != null) {
                doc.add(new TextField("author", p.matcher(song.getAuthor()).replaceAll(""), Field.Store.NO));
            }
            if (song.getLyrics(false, false, false) != null) {
                doc.add(new TextField("lyrics", p.matcher(song.getLyrics(false, false, false)).replaceAll(""), Field.Store.NO));
            }
            doc.add(new TextField("number", p.matcher(Long.toString(song.getID())).replaceAll(""), Field.Store.YES));
            writer.addDocument(doc);
            songs.put(song.getID(), song);
            LOGGER.log(Level.FINE, "Added song to index: {0}", song.getTitle());
        }
    } catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 
Example #5
Source File: TestIndexWriterExceptions.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** test a null bytesref value doesn't abort the entire segment */
public void testNullStoredBytesRefFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", new BytesRef("hello"));
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    BytesRef v = null;
    theField.setBytesValue(v);
    iw.addDocument(doc);
    fail("didn't get expected exception");
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 
Example #6
Source File: TestTermVectorsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNoAbortOnBadTVSettings() throws Exception {
  Directory dir = newDirectory();
  // Don't use RandomIndexWriter because we want to be sure both docs go to 1 seg:
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter iw = new IndexWriter(dir, iwc);

  Document doc = new Document();
  iw.addDocument(doc);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectors(true);
  ft.freeze();
  doc.add(new Field("field", "value", ft));

  expectThrows(IllegalArgumentException.class, () -> {
    iw.addDocument(doc);
  });

  IndexReader r = DirectoryReader.open(iw);

  // Make sure the exc didn't lose our first document:
  assertEquals(1, r.numDocs());
  iw.close();
  r.close();
  dir.close();
}
 
Example #7
Source File: TestTransactionRollback.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();

  //Build index, of records 1 to 100, committing after each batch of 10
  IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
  IndexWriter w=new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                        .setIndexDeletionPolicy(sdp));

  for(int currentRecordId=1;currentRecordId<=100;currentRecordId++) {
    Document doc=new Document();
    doc.add(newTextField(FIELD_RECORD_ID, ""+currentRecordId, Field.Store.YES));
    w.addDocument(doc);

    if (currentRecordId%10 == 0) {
      Map<String,String> data = new HashMap<>();
      data.put("index", "records 1-"+currentRecordId);
      w.setLiveCommitData(data.entrySet());
      w.commit();
    }
  }

  w.close();
}
 
Example #8
Source File: LicenseIndexer.java    From dependency-track with Apache License 2.0 6 votes vote down vote up
/**
 * Adds a License object to a Lucene index.
 *
 * @param license A persisted License object.
 */
public void add(final License license) {
    final Document doc = new Document();
    addField(doc, IndexConstants.LICENSE_UUID, license.getUuid().toString(), Field.Store.YES, false);
    addField(doc, IndexConstants.LICENSE_LICENSEID, license.getLicenseId(), Field.Store.YES, true);
    addField(doc, IndexConstants.LICENSE_NAME, license.getName(), Field.Store.YES, true);

    try {
        getIndexWriter().addDocument(doc);
    } catch (IOException e) {
        LOGGER.error("An error occurred while adding a license to the index", e);
        Notification.dispatch(new Notification()
                .scope(NotificationScope.SYSTEM)
                .group(NotificationGroup.INDEXING_SERVICE)
                .title(NotificationConstants.Title.LICENSE_INDEXER)
                .content("An error occurred while adding a license to the index. Check log for details. " + e.getMessage())
                .level(NotificationLevel.ERROR)
        );
    }
}
 
Example #9
Source File: InMemoryIndex.java    From SnowGraph with Apache License 2.0 6 votes vote down vote up
public InMemoryIndex(Map<String,String> id2Text){
    Analyzer analyzer = new EnglishAnalyzer();
    IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    try {
        IndexWriter writer = new IndexWriter(directory, iwc);
        for (String id:id2Text.keySet()) {
            Document doc=new Document();
            doc.add(new StringField("id", id, Field.Store.YES));
            doc.add(new TextField("content", id2Text.get(id), Field.Store.YES));
            writer.addDocument(doc);
        }
        writer.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example #10
Source File: TestConstantScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testPropagatesApproximations() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  Field f = newTextField("field", "a b", Field.Store.NO);
  doc.add(f);
  w.addDocument(doc);
  w.commit();

  DirectoryReader reader = w.getReader();
  final IndexSearcher searcher = newSearcher(reader);
  searcher.setQueryCache(null); // to still have approximations

  PhraseQuery pq = new PhraseQuery("field", "a", "b");

  Query q = searcher.rewrite(new ConstantScoreQuery(pq));

  final Weight weight = searcher.createWeight(q, ScoreMode.COMPLETE, 1);
  final Scorer scorer = weight.scorer(searcher.getIndexReader().leaves().get(0));
  assertNotNull(scorer.twoPhaseIterator());

  reader.close();
  w.close();
  dir.close();
}
 
Example #11
Source File: TestTermVectorsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIllegalVectorOffsetsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorOffsets(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector offsets for a field that is not indexed (field=\"field\")", expected.getMessage());
  
  w.close();
  dir.close();
}
 
Example #12
Source File: TestSimilarity2.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** similar to the above, however the field exists, but we query with a term that doesnt exist too */
public void testEmptyTerm() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("foo", "bar", Field.Store.NO));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);
  
  for (Similarity sim : sims) {
    is.setSimilarity(sim);
    BooleanQuery.Builder query = new BooleanQuery.Builder();
    query.add(new TermQuery(new Term("foo", "bar")), BooleanClause.Occur.SHOULD);
    query.add(new TermQuery(new Term("foo", "baz")), BooleanClause.Occur.SHOULD);
    assertEquals(1, is.search(query.build(), 10).totalHits.value);
  }
  ir.close();
  dir.close();
}
 
Example #13
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testExcIndexingDocBeforeDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setDocValuesType(DocValuesType.SORTED);
  ft.freeze();
  Field field = new Field("test", "value", ft);
  field.setTokenStream(new TokenStream() {
      @Override
      public boolean incrementToken() {
        throw new RuntimeException("no");
      }
    });
  doc.add(field);
  expectThrows(RuntimeException.class, () -> {
    w.addDocument(doc);
  });

  w.addDocument(new Document());
  w.close();
  dir.close();
}
 
Example #14
Source File: EngineTestCase.java    From crate with Apache License 2.0 6 votes vote down vote up
protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate,
    boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}
 
Example #15
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNRTReaderVersion() throws Exception {
  Directory d = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
  IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  doc.add(newStringField("id", "0", Field.Store.YES));
  w.addDocument(doc);
  DirectoryReader r = w.getReader();
  long version = r.getVersion();
  r.close();

  w.addDocument(doc);
  r = w.getReader();
  long version2 = r.getVersion();
  r.close();
  assert(version2 > version);

  w.deleteDocuments(new Term("id", "0"));
  r = w.getReader();
  w.close();
  long version3 = r.getVersion();
  r.close();
  assert(version3 > version2);
  d.close();
}
 
Example #16
Source File: TokenCountFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = StringFieldMapper.parseCreateFieldForString(context, null /* Out null value is an int so we convert*/, fieldType().boost());
    if (valueAndBoost.value() == null && fieldType().nullValue() == null) {
        return;
    }

    if (fieldType().indexOptions() != NONE || fieldType().stored() || fieldType().hasDocValues()) {
        int count;
        if (valueAndBoost.value() == null) {
            count = fieldType().nullValue();
        } else {
            count = countPositions(analyzer, simpleName(), valueAndBoost.value());
        }
        addIntegerFields(context, fields, count, valueAndBoost.boost());
    }
}
 
Example #17
Source File: TestFieldCacheSort.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** test that we throw exception on multi-valued field, creates corrupt reader, use SORTED_SET instead */
public void testMultiValuedField() throws IOException {
  Directory indexStore = newDirectory();
  IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(new MockAnalyzer(random())));
  for(int i=0; i<5; i++) {
      Document doc = new Document();
      doc.add(new StringField("string", "a"+i, Field.Store.NO));
      doc.add(new StringField("string", "b"+i, Field.Store.NO));
      writer.addDocument(doc);
  }
  writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
  writer.close();
  Sort sort = new Sort(
      new SortField("string", SortField.Type.STRING),
      SortField.FIELD_DOC);
  IndexReader reader = UninvertingReader.wrap(DirectoryReader.open(indexStore),
                       Collections.singletonMap("string", Type.SORTED));
  IndexSearcher searcher = new IndexSearcher(reader);
  expectThrows(IllegalStateException.class, () -> {
    searcher.search(new MatchAllDocsQuery(), 500, sort);
  });
  reader.close();
  indexStore.close();
}
 
Example #18
Source File: TestExpressionSorts.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(500);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example #19
Source File: TestUnifiedHighlighterTermIntervals.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Make sure highlighter returns first N sentences if
 * there were no hits.
 */
public void testEmptyHighlights() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Document doc = new Document();

  Field body = new Field("body", "test this is.  another sentence this test has.  far away is that planet.", fieldType);
  doc.add(body);
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  Query query = new IntervalQuery("body", Intervals.term("highlighting"));
  int[] docIDs = new int[] {0};
  String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");
  assertEquals(1, snippets.length);
  assertEquals("test this is.  another sentence this test has.  ", snippets[0]);
  ir.close();
}
 
Example #20
Source File: TestIndexWriterExceptions.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** test a null string value doesn't abort the entire segment */
public void testNullStoredFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", "hello", StoredField.TYPE);
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    theField.setStringValue(null);
    iw.addDocument(doc);
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 
Example #21
Source File: TestUnifiedHighlighterTermIntervals.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testBasics() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Field body = new Field("body", "", fieldType);
  Document doc = new Document();
  doc.add(body);

  body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
  iw.addDocument(doc);
  body.setStringValue("Highlighting the first term. Hope it works.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  Query query = new IntervalQuery("body", Intervals.term("highlighting"));
  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  assertEquals(2, topDocs.totalHits.value);
  String snippets[] = highlighter.highlight("body", query, topDocs);
  assertEquals(2, snippets.length);
  assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
  assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
  ir.close();
}
 
Example #22
Source File: TestLongValuesSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = TestUtil.nextInt(random(), 2049, 4000);
  int leastValue = 45;
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    if (i == 545)
      document.add(new NumericDocValuesField("onefield", LEAST_LONG_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example #23
Source File: TestDirectSpellChecker.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testBogusField() throws Exception {
  DirectSpellChecker spellChecker = new DirectSpellChecker();
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer);

  for (int i = 0; i < 20; i++) {
    Document doc = new Document();
    doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO));
    writer.addDocument(doc);
  }

  IndexReader ir = writer.getReader();

  SuggestWord[] similar = spellChecker.suggestSimilar(new Term(
      "bogusFieldBogusField", "fvie"), 2, ir,
      SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
  assertEquals(0, similar.length);
  
  IOUtils.close(ir, writer, dir, analyzer);
}
 
Example #24
Source File: TestPostingsOffsets.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void checkTokens(Token[] field1, Token[] field2) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);
   
    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(field1), ft));
    doc.add(new Field("body", new CannedTokenStream(field2), ft));
    riw.addDocument(doc);
    riw.close();
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}
 
Example #25
Source File: TestRegexpQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
  Document doc = new Document();
  doc.add(newTextField(FN, "the quick brown fox jumps over the lazy ??? dog 493432 49344 [foo] 12.3 \\", Field.Store.NO));
  writer.addDocument(doc);
  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(reader);
}
 
Example #26
Source File: ReferenceCountingReadOnlyIndexReaderFactory.java    From alfresco-repository with GNU Lesser General Public License v3.0 5 votes vote down vote up
public String getType(int n) throws IOException
{
    // return getStringValue(n, "TYPE");
    Document d = document(n, new SingleFieldSelector("TYPE", true));
    Field f = d.getField("TYPE");
    return f == null ? null : f.stringValue();
}
 
Example #27
Source File: ReferenceCountingReadOnlyIndexReaderFactory.java    From alfresco-repository with GNU Lesser General Public License v3.0 5 votes vote down vote up
public String getPathLinkId(int n) throws IOException
{
    Document document = document(n, new SingleFieldSelector("ID", true));
    Field[] fields = document.getFields("ID");
    Field field = fields[fields.length - 1];
    return (field == null) ? null : field.stringValue();
}
 
Example #28
Source File: TestSimpleMatcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testSimpleMatcher() throws IOException {

    try (Monitor monitor = newMonitor()) {
      monitor.register(
          new MonitorQuery("1", parse("test")),
          new MonitorQuery("2", parse("wibble")));
      Document doc = new Document();
      doc.add(newTextField(FIELD, "test", Field.Store.NO));

      MatchingQueries<QueryMatch> matches = monitor.match(doc, QueryMatch.SIMPLE_MATCHER);
      assertNotNull(matches.matches("1"));
    }
  }
 
Example #29
Source File: TestCustomTermFreq.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testInvalidProx() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));

  Document doc = new Document();
  FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
  Field field = new Field("field",
                          new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"},
                                              new int[] {42, 128, 17, 100}),
                          fieldType);
  doc.add(field);
  Exception e = expectThrows(IllegalStateException.class, () -> {w.addDocument(doc);});
  assertEquals("field \"field\": cannot index positions while using custom TermFrequencyAttribute", e.getMessage());
  IOUtils.close(w, dir);
}
 
Example #30
Source File: BaseDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testSortedMergeAwayAllValuesLargeSegment() throws IOException {
  Directory directory = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
  iwconfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);

  Document doc = new Document();
  doc.add(new StringField("id", "1", Field.Store.NO));
  doc.add(new SortedDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  final int numEmptyDocs = atLeast(1024);
  for (int i = 0; i < numEmptyDocs; ++i) {
    iwriter.addDocument(new Document());
  }
  iwriter.commit();
  iwriter.deleteDocuments(new Term("id", "1"));
  iwriter.forceMerge(1);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field");
  assertEquals(NO_MORE_DOCS, dv.nextDoc());

  ireader.close();
  directory.close();
}