org.apache.lucene.document.FieldType Java Examples

The following examples show how to use org.apache.lucene.document.FieldType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OLuceneSpatialIndexManager.java    From orientdb-lucene with Apache License 2.0 6 votes vote down vote up
private Document newGeoDocument(OIdentifiable oIdentifiable, Shape shape) {

    FieldType ft = new FieldType();
    ft.setIndexed(true);
    ft.setStored(true);

    Document doc = new Document();

    doc.add(OLuceneIndexType.createField(RID, oIdentifiable.getIdentity().toString(), Field.Store.YES,
        Field.Index.NOT_ANALYZED_NO_NORMS));
    for (IndexableField f : strategy.createIndexableFields(shape)) {
      doc.add(f);
    }

    doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));

    return doc;
  }
 
Example #2
Source File: TestSloppyPhraseQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testInfiniteFreq1() throws Exception {
  String document = "drug druggy drug drug drug";
  
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newField("lyrics", document, new FieldType(TextField.TYPE_NOT_STORED)));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  
  IndexSearcher is = newSearcher(ir);
  PhraseQuery.Builder builder = new PhraseQuery.Builder();
  builder.add(new Term("lyrics", "drug"), 1);
  builder.add(new Term("lyrics", "drug"), 3);
  builder.setSlop(1);
  PhraseQuery pq = builder.build();
  // "drug the drug"~1
  assertSaneScoring(pq, is);
  ir.close();
  dir.close();
}
 
Example #3
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testExcIndexingDocBeforeDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setDocValuesType(DocValuesType.SORTED);
  ft.freeze();
  Field field = new Field("test", "value", ft);
  field.setTokenStream(new TokenStream() {
      @Override
      public boolean incrementToken() {
        throw new RuntimeException("no");
      }
    });
  doc.add(field);
  expectThrows(RuntimeException.class, () -> {
    w.addDocument(doc);
  });

  w.addDocument(new Document());
  w.close();
  dir.close();
}
 
Example #4
Source File: TestTermVectorsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIllegalVectorPositionsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorPositions(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector positions for a field that is not indexed (field=\"field\")", expected.getMessage());
  
  w.close();
  dir.close();
}
 
Example #5
Source File: AnalyzingInfixSuggester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 
Example #6
Source File: FilePositionDoc.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public static Document Document(File f)
     throws java.io.FileNotFoundException {
  Document doc = new Document();
  doc.add(new StoredField("path", f.getPath()));
  doc.add(new StoredField("modified",
                    DateTools.timeToString(f.lastModified(), DateTools.Resolution.MINUTE)));
  
  //create new FieldType to store term positions (TextField is not sufficiently configurable)
  FieldType ft = new FieldType();
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  ft.setTokenized(true);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  Field contentsField = new Field("contents", new FileReader(f), ft);

  doc.add(contentsField);
  return doc;
}
 
Example #7
Source File: TestCustomTermFreq.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testFieldInvertState() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  iwc.setSimilarity(NeverForgetsSimilarity.INSTANCE);
  IndexWriter w = new IndexWriter(dir, iwc);

  Document doc = new Document();
  FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
  fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  Field field = new Field("field",
                          new CannedTermFreqs(new String[] {"foo", "bar", "foo", "bar"},
                                              new int[] {42, 128, 17, 100}),
                          fieldType);
  doc.add(field);
  w.addDocument(doc);
  FieldInvertState fis = NeverForgetsSimilarity.INSTANCE.lastState;
  assertEquals(228, fis.getMaxTermFrequency());
  assertEquals(2, fis.getUniqueTermCount());
  assertEquals(0, fis.getNumOverlap());
  assertEquals(287, fis.getLength());

  IOUtils.close(w, dir);
}
 
Example #8
Source File: TestTermVectorsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNoAbortOnBadTVSettings() throws Exception {
  Directory dir = newDirectory();
  // Don't use RandomIndexWriter because we want to be sure both docs go to 1 seg:
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter iw = new IndexWriter(dir, iwc);

  Document doc = new Document();
  iw.addDocument(doc);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectors(true);
  ft.freeze();
  doc.add(new Field("field", "value", ft));

  expectThrows(IllegalArgumentException.class, () -> {
    iw.addDocument(doc);
  });

  IndexReader r = DirectoryReader.open(iw);

  // Make sure the exc didn't lose our first document:
  assertEquals(1, r.numDocs());
  iw.close();
  r.close();
  dir.close();
}
 
Example #9
Source File: DocHelper.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static Document createDocument(int n, String indexName, int numFields) {
  StringBuilder sb = new StringBuilder();
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(true);
  customType.setStoreTermVectorOffsets(true);

  FieldType customType1 = new FieldType(StringField.TYPE_STORED);
  customType1.setStoreTermVectors(true);
  customType1.setStoreTermVectorPositions(true);
  customType1.setStoreTermVectorOffsets(true);

  final Document doc = new Document();
  doc.add(new Field("id", Integer.toString(n), customType1));
  doc.add(new Field("indexname", indexName, customType1));
  sb.append("a");
  sb.append(n);
  doc.add(new Field("field1", sb.toString(), customType));
  sb.append(" b");
  sb.append(n);
  for (int i = 1; i < numFields; i++) {
    doc.add(new Field("field" + (i + 1), sb.toString(), customType));
  }
  return doc;
}
 
Example #10
Source File: TestPostingsOffsets.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testLegalbutVeryLargeOffsets() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  Token t1 = new Token("foo", 0, Integer.MAX_VALUE-500);
  if (random().nextBoolean()) {
    t1.setPayload(new BytesRef("test"));
  }
  Token t2 = new Token("foo", Integer.MAX_VALUE-500, Integer.MAX_VALUE);
  TokenStream tokenStream = new CannedTokenStream(
      new Token[] { t1, t2 }
  );
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  // store some term vectors for the checkindex cross-check
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  ft.setStoreTermVectorOffsets(true);
  Field field = new Field("foo", tokenStream, ft);
  doc.add(field);
  iw.addDocument(doc);
  iw.close();
  dir.close();
}
 
Example #11
Source File: TestUnifiedHighlighterTermIntervals.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private IndexReader indexSomeFields() throws IOException {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
  FieldType ft = new FieldType();
  ft.setIndexOptions(IndexOptions.NONE);
  ft.setTokenized(false);
  ft.setStored(true);
  ft.freeze();

  Field title = new Field("title", "", fieldType);
  Field text = new Field("text", "", fieldType);
  Field category = new Field("category", "", fieldType);

  Document doc = new Document();
  doc.add(title);
  doc.add(text);
  doc.add(category);
  title.setStringValue("This is the title field.");
  text.setStringValue("This is the text field. You can put some text if you want.");
  category.setStringValue("This is the category field.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();
  return ir;
}
 
Example #12
Source File: TestUnifiedHighlighter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private IndexReader indexSomeFields() throws IOException {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
  FieldType ft = new FieldType();
  ft.setIndexOptions(IndexOptions.NONE);
  ft.setTokenized(false);
  ft.setStored(true);
  ft.freeze();

  Field title = new Field("title", "", fieldType);
  Field text = new Field("text", "", fieldType);
  Field category = new Field("category", "", fieldType);

  Document doc = new Document();
  doc.add(title);
  doc.add(text);
  doc.add(category);
  title.setStringValue("This is the title field.");
  text.setStringValue("This is the text field. You can put some text if you want.");
  category.setStringValue("This is the category field.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();
  return ir;
}
 
Example #13
Source File: DocMaker.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public DocState(boolean reuseFields, FieldType ft, FieldType bodyFt) {

      this.reuseFields = reuseFields;
      
      if (reuseFields) {
        fields =  new HashMap<>();
        numericFields = new HashMap<>();
        
        // Initialize the map with the default fields.
        fields.put(BODY_FIELD, new Field(BODY_FIELD, "", bodyFt));
        fields.put(TITLE_FIELD, new Field(TITLE_FIELD, "", ft));
        fields.put(DATE_FIELD, new Field(DATE_FIELD, "", ft));
        fields.put(ID_FIELD, new StringField(ID_FIELD, "", Field.Store.YES));
        fields.put(NAME_FIELD, new Field(NAME_FIELD, "", ft));

        numericFields.put(DATE_MSEC_FIELD, new LongPoint(DATE_MSEC_FIELD, 0L));
        numericFields.put(TIME_SEC_FIELD, new IntPoint(TIME_SEC_FIELD, 0));
        
        doc = new Document();
      } else {
        numericFields = null;
        fields = null;
        doc = null;
      }
    }
 
Example #14
Source File: TestMemoryIndex.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIndexingPointsAndDocValues() throws Exception {
  FieldType type = new FieldType();
  type.setDimensions(1, 4);
  type.setDocValuesType(DocValuesType.BINARY);
  type.freeze();
  Document doc = new Document();
  byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8);
  doc.add(new BinaryPoint("field", packedPoint, type));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  assertEquals(1, leafReader.getPointValues("field").size());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMinPackedValue());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMaxPackedValue());

  BinaryDocValues dvs = leafReader.getBinaryDocValues("field");
  assertEquals(0, dvs.nextDoc());
  assertEquals("term", dvs.binaryValue().utf8ToString());
}
 
Example #15
Source File: TermDocIterableTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private void addDocumentBlock(int id, int count, IndexWriter writer) throws IOException {
  FieldType fieldType = new FieldType();
  fieldType.setIndexed(true);
  fieldType.setOmitNorms(true);
  fieldType.setTokenized(false);
  fieldType.setStored(true);

  FieldType fieldTypeNoIndex = new FieldType();
  fieldTypeNoIndex.setStored(true);
  fieldTypeNoIndex.setIndexed(false);

  for (int i = 0; i < count; i++) {
    Document document = new Document();
    document.add(new Field("id", Integer.toString(id), fieldType));
    document.add(new Field("field", Integer.toString(i), fieldType));
    for (int j = 0; j < 100; j++) {
      document.add(new Field("field" + j, "testing here testing here testing here testing here testing here testing here testing here", fieldTypeNoIndex));
    }
    writer.addDocument(document);
  }
}
 
Example #16
Source File: LongFieldTypeDefinition.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(String fieldNameForThisInstance, Map<String, String> properties, Configuration configuration) {
  String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP);
  if (precisionStepStr != null) {
    _precisionStep = Integer.parseInt(precisionStepStr);
    _typeStored = new FieldType(LongField.TYPE_STORED);
    _typeStored.setNumericPrecisionStep(_precisionStep);
    _typeStored.freeze();
    _typeNotStored = new FieldType(LongField.TYPE_NOT_STORED);
    _typeNotStored.setNumericPrecisionStep(_precisionStep);
    _typeNotStored.freeze();
  } else {
    _typeStored = LongField.TYPE_STORED;
    _typeNotStored = LongField.TYPE_NOT_STORED;
  }
}
 
Example #17
Source File: PresearcherTestBase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNonStringTermHandling() throws IOException {

    FieldType ft = new FieldType();
    ft.setTokenized(true);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);

    try (Monitor monitor = newMonitor()) {
      monitor.register(new MonitorQuery("1", new TermQuery(new Term("f", NON_STRING_TERM))));

      Document doc = new Document();
      doc.add(new Field("f", new NonStringTokenStream(), ft));
      MatchingQueries<QueryMatch> m = monitor.match(doc, QueryMatch.SIMPLE_MATCHER);
      assertEquals(1, m.getMatchCount());
      assertEquals(1, m.getQueriesRun());
    }

  }
 
Example #18
Source File: FieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static String termVectorOptionsToString(FieldType fieldType) {
    if (!fieldType.storeTermVectors()) {
        return "no";
    } else if (!fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "yes";
    } else if (fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "with_offsets";
    } else {
        StringBuilder builder = new StringBuilder("with");
        if (fieldType.storeTermVectorPositions()) {
            builder.append("_positions");
        }
        if (fieldType.storeTermVectorOffsets()) {
            builder.append("_offsets");
        }
        if (fieldType.storeTermVectorPayloads()) {
            builder.append("_payloads");
        }
        return builder.toString();
    }
}
 
Example #19
Source File: TestBlockPostingsFormat2.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private Document newDocument() {
  Document doc = new Document();
  for (IndexOptions option : IndexOptions.values()) {
    if (option == IndexOptions.NONE) {
      continue;
    }
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    // turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setIndexOptions(option);
    doc.add(new Field(option.toString(), "", ft));
  }
  return doc;
}
 
Example #20
Source File: BlendedInfixSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected FieldType getTextFieldType() {
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  ft.setOmitNorms(true);

  return ft;
}
 
Example #21
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings));
	mappers.mutator().putAll(fieldMappers).close();
	this.threadPool = threadPool;
}
 
Example #22
Source File: LuceneTestCase.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static Field createField(String name, Object value, FieldType fieldType) {
  if (value instanceof String) {
    return new Field(name, (String) value, fieldType);
  } else if (value instanceof BytesRef) {
    return new Field(name, (BytesRef) value, fieldType);
  } else {
    throw new IllegalArgumentException("value must be String or BytesRef");
  }
}
 
Example #23
Source File: LtrQueryTests.java    From elasticsearch-learning-to-rank with Apache License 2.0 5 votes vote down vote up
private Field newField(String name, String value, Store stored) {
    FieldType tagsFieldType = new FieldType();
    tagsFieldType.setStored(stored == Store.YES);
    IndexOptions idxOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
    tagsFieldType.setIndexOptions(idxOptions);
    return new Field(name, value, tagsFieldType);
}
 
Example #24
Source File: LineFileDocs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public DocState() {
  doc = new Document();
  
  title = new StringField("title", "", Field.Store.NO);
  doc.add(title);

  FieldType ft = new FieldType(TextField.TYPE_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  
  titleTokenized = new Field("titleTokenized", "", ft);
  doc.add(titleTokenized);

  body = new Field("body", "", ft);
  doc.add(body);

  id = new StringField("docid", "", Field.Store.YES);
  doc.add(id);

  idNum = new IntPoint("docid_int", 0);
  doc.add(idNum);

  date = new StringField("date", "", Field.Store.YES);
  doc.add(date);

  titleDV = new SortedDocValuesField("titleDV", new BytesRef());
  idNumDV = new NumericDocValuesField("docid_intDV", 0);
  doc.add(titleDV);
  doc.add(idNumDV);
}
 
Example #25
Source File: TestOmitNorms.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Tests various combinations of omitNorms=true/false, the field not existing at all,
 * ensuring that only omitNorms is 'viral'.
 * Internally checks that MultiNorms.norms() is consistent (returns the same bytes)
 * as the fully merged equivalent.
 */
public void testOmitNormsCombos() throws IOException {
  // indexed with norms
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  Field norms = new Field("foo", "a", customType);
  // indexed without norms
  FieldType customType1 = new FieldType(TextField.TYPE_STORED);
  customType1.setOmitNorms(true);
  Field noNorms = new Field("foo", "a", customType1);
  // not indexed, but stored
  FieldType customType2 = new FieldType();
  customType2.setStored(true);
  Field noIndex = new Field("foo", "a", customType2);
  // not indexed but stored, omitNorms is set
  FieldType customType3 = new FieldType();
  customType3.setStored(true);
  customType3.setOmitNorms(true);
  Field noNormsNoIndex = new Field("foo", "a", customType3);
  // not indexed nor stored (doesnt exist at all, we index a different field instead)
  Field emptyNorms = new Field("bar", "a", customType);
  
  assertNotNull(getNorms("foo", norms, norms));
  assertNull(getNorms("foo", norms, noNorms));
  assertNotNull(getNorms("foo", norms, noIndex));
  assertNotNull(getNorms("foo", norms, noNormsNoIndex));
  assertNotNull(getNorms("foo", norms, emptyNorms));
  assertNull(getNorms("foo", noNorms, noNorms));
  assertNull(getNorms("foo", noNorms, noIndex));
  assertNull(getNorms("foo", noNorms, noNormsNoIndex));
  assertNull(getNorms("foo", noNorms, emptyNorms));
  assertNull(getNorms("foo", noIndex, noIndex));
  assertNull(getNorms("foo", noIndex, noNormsNoIndex));
  assertNull(getNorms("foo", noIndex, emptyNorms));
  assertNull(getNorms("foo", noNormsNoIndex, noNormsNoIndex));
  assertNull(getNorms("foo", noNormsNoIndex, emptyNorms));
  assertNull(getNorms("foo", emptyNorms, emptyNorms));
}
 
Example #26
Source File: TermVectorsAdapterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  FieldType textType = new FieldType();
  textType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType.setTokenized(true);
  textType.setStoreTermVectors(true);

  FieldType textType_pos = new FieldType();
  textType_pos.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType_pos.setTokenized(true);
  textType_pos.setStoreTermVectors(true);
  textType_pos.setStoreTermVectorPositions(true);

  FieldType textType_pos_offset = new FieldType();
  textType_pos_offset.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType_pos_offset.setTokenized(true);
  textType_pos_offset.setStoreTermVectors(true);
  textType_pos_offset.setStoreTermVectorPositions(true);
  textType_pos_offset.setStoreTermVectorOffsets(true);

  String text = "It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife.";
  Document doc = new Document();
  doc.add(newField("text1", text, textType));
  doc.add(newField("text2", text, textType_pos));
  doc.add(newField("text3", text, textType_pos_offset));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 
Example #27
Source File: LuceneIndexer.java    From MtgDesktopCompanion with GNU General Public License v3.0 5 votes vote down vote up
private Document toDocuments(MagicCard mc) {
       Document doc = new Document();
       			
       		FieldType fieldType = new FieldType();
         		fieldType.setStored(true);
         		fieldType.setStoreTermVectors(true);
         		fieldType.setTokenized(true);
         		fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
         		
        		   doc.add(new Field("name", mc.getName(), fieldType));
        		   
        		   if(mc.getCost()!=null)
        			   doc.add(new Field("cost", mc.getCost(),fieldType));
        		   else
        			   doc.add(new Field("cost", "",fieldType));
        		  
        		   if(mc.getText()!=null)
        			   doc.add(new Field("text", mc.getText(), fieldType));
        		   else
        			   doc.add(new Field("text", "", fieldType));
        		   
        		   doc.add(new Field("type", mc.getFullType(), fieldType));
        		   doc.add(new Field("set",mc.getCurrentSet().getId(),fieldType));
        		   doc.add(new StoredField("cmc",mc.getCmc()));
        		   doc.add(new StringField("data",serializer.toJson(mc),Field.Store.YES));
        		   
         	   for(MTGColor color:mc.getColors())
         	   {
         		   doc.add(new Field("color", color.getCode(), fieldType));
         	   }
         	 
         	   
    		   
      return doc;
}
 
Example #28
Source File: TestOrdValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void addDoc(RandomIndexWriter iw, int i) throws Exception {
  Document d = new Document();
  Field f;
  int scoreAndID = i + 1;

  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setTokenized(false);
  customType.setOmitNorms(true);
  
  f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
  d.add(f);
  d.add(new SortedDocValuesField(ID_FIELD, new BytesRef(id2String(scoreAndID))));

  FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
  customType2.setOmitNorms(true);
  f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
  d.add(f);

  f = new LegacyIntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
  d.add(f);
  d.add(new NumericDocValuesField(INT_FIELD, scoreAndID));

  f = new LegacyFloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
  d.add(f);
  d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID)));

  iw.addDocument(d);
  log("added: " + d);
}
 
Example #29
Source File: TestBBoxStrategy.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testOperations() throws IOException {
  //setup
  if (random().nextInt(4) > 0) {//75% of the time choose geo (more interesting to test)
    this.ctx = SpatialContext.GEO;
  } else {
    SpatialContextFactory factory = new SpatialContextFactory();
    factory.geo = false;
    factory.worldBounds = new RectangleImpl(-300, 300, -100, 100, null);
    this.ctx = factory.newSpatialContext();
  }
  this.strategy = BBoxStrategy.newInstance(ctx, "bbox");
  //test we can disable docValues for predicate tests
  if (random().nextBoolean()) {
    FieldType fieldType = new FieldType(((BBoxStrategy)strategy).getFieldType());
    fieldType.setDocValuesType(DocValuesType.NONE);
    strategy = new BBoxStrategy(ctx, strategy.getFieldName(), fieldType);
  }
  for (SpatialOperation operation : SpatialOperation.values()) {
    if (operation == SpatialOperation.Overlaps)
      continue;//unsupported
    testOperationRandomShapes(operation);

    deleteAll();
    commit();
  }
}
 
Example #30
Source File: BBoxStrategy.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Creates this strategy.
 * {@code fieldType} is used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL
 * field too. Search requires pointValues (or legacy numerics), and relevancy requires docValues. If these features
 * aren't needed then disable them.
 */
public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  field_bbox = fieldNamePrefix;
  field_minX = fieldNamePrefix + SUFFIX_MINX;
  field_maxX = fieldNamePrefix + SUFFIX_MAXX;
  field_minY = fieldNamePrefix + SUFFIX_MINY;
  field_maxY = fieldNamePrefix + SUFFIX_MAXY;
  field_xdl = fieldNamePrefix + SUFFIX_XDL;

  fieldType.freeze();
  this.optionsFieldType = fieldType;

  int numQuads = 0;
  if ((this.hasStored = fieldType.stored())) {
    numQuads++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numQuads++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numQuads++;
  }

  if (hasPointVals) { // if we have an index...
    xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED);
    xdlFieldType.setIndexOptions(IndexOptions.DOCS);
    xdlFieldType.freeze();
  } else {
    xdlFieldType = null;
  }

  this.fieldsLen = numQuads * 4 + (xdlFieldType != null ? 1 : 0);
}