Java Code Examples for org.apache.lucene.index.IndexableField

The following examples show how to use org.apache.lucene.index.IndexableField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: CurrencyFieldType.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object externalVal) {
  CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);

  List<IndexableField> f = new ArrayList<>();
  SchemaField amountField = getAmountField(field);
  f.add(amountField.createField(String.valueOf(value.getAmount())));
  SchemaField currencyField = getCurrencyField(field);
  f.add(currencyField.createField(value.getCurrencyCode()));

  if (field.stored()) {
    String storedValue = externalVal.toString().trim();
    if (storedValue.indexOf(",") < 0) {
      storedValue += "," + defaultCurrency;
    }
    f.add(createField(field.getName(), storedValue, StoredField.TYPE));
  }

  return f;
}
 
Example 2
Source Project: lucene-solr   Source File: TestSortableTextField.java    License: Apache License 2.0 6 votes vote down vote up
public void testWhiteboxCreateFields() throws Exception {
  List<IndexableField> values = null;

  // common case...
  for (String field : Arrays.asList("keyword_stxt", "keyword_dv_stxt",
                                    "whitespace_stxt", "whitespace_f_stxt", "whitespace_l_stxt")) {
    values = createIndexableFields(field);
    assertEquals(field, 2, values.size());
    assertThat(field, values.get(0), instanceOf(Field.class));
    assertThat(field, values.get(1), instanceOf(SortedDocValuesField.class));
  }
  
  // special cases...
  values = createIndexableFields("whitespace_nois_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(SortedDocValuesField.class));
  //
  values = createIndexableFields("whitespace_nodv_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  //
  values = createIndexableFields("whitespace_m_stxt");
  assertEquals(2, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  assertThat(values.get(1), instanceOf(SortedSetDocValuesField.class));      
}
 
Example 3
Source Project: lucene-solr   Source File: LazyDocument.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a StorableField whose value will be lazy loaded if and 
 * when it is used. 
 * <p>
 * <b>NOTE:</b> This method must be called once for each value of the field 
 * name specified in sequence that the values exist.  This method may not be 
 * used to generate multiple, lazy, StorableField instances referring to 
 * the same underlying StorableField instance.
 * </p>
 * <p>
 * The lazy loading of field values from all instances of StorableField 
 * objects returned by this method are all backed by a single StoredDocument 
 * per LazyDocument instance.
 * </p>
 */
public IndexableField getField(FieldInfo fieldInfo) {  

  fieldNames.add(fieldInfo.name);
  List<LazyField> values = fields.get(fieldInfo.number);
  if (null == values) {
    values = new ArrayList<>();
    fields.put(fieldInfo.number, values);
  } 

  LazyField value = new LazyField(fieldInfo.name, fieldInfo.number);
  values.add(value);

  synchronized (this) {
    // edge case: if someone asks this LazyDoc for more LazyFields
    // after other LazyFields from the same LazyDoc have been
    // actuallized, we need to force the doc to be re-fetched
    // so the new LazyFields are also populated.
    doc = null;
  }
  return value;
}
 
Example 4
Source Project: jstarcraft-core   Source File: ArrayStoreConverter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public NavigableMap<String, IndexableField> encode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, Object instance) {
    NavigableMap<String, IndexableField> indexables = new TreeMap<>();
    Class<?> componentClass = null;
    Type componentType = null;
    if (type instanceof GenericArrayType) {
        GenericArrayType genericArrayType = GenericArrayType.class.cast(type);
        componentType = genericArrayType.getGenericComponentType();
        componentClass = TypeUtility.getRawType(componentType, null);
    } else {
        Class<?> clazz = TypeUtility.getRawType(type, null);
        componentType = clazz.getComponentType();
        componentClass = clazz.getComponentType();
    }
    Specification specification = Specification.getSpecification(componentClass);
    StoreConverter converter = context.getStoreConverter(specification);
    int size = Array.getLength(instance);
    IndexableField indexable = new StoredField(path + ".size", size);
    indexables.put(path + ".size", indexable);
    for (int index = 0; index < size; index++) {
        Object element = Array.get(instance, index);
        indexables.putAll(converter.encode(context, path + "[" + index + "]", field, annotation, componentType, element));
    }
    return indexables;
}
 
Example 5
Source Project: nifi   Source File: DocsReader.java    License: Apache License 2.0 6 votes vote down vote up
private ProvenanceEventRecord getRecord(final Document d, final RecordReader reader) throws IOException {
    final IndexableField blockField = d.getField(FieldNames.BLOCK_INDEX);
    if ( blockField == null ) {
        reader.skipTo(getByteOffset(d, reader));
    } else {
        reader.skipToBlock(blockField.numericValue().intValue());
    }

    StandardProvenanceEventRecord record;
    while ( (record = reader.nextRecord()) != null) {
        final IndexableField idField = d.getField(SearchableFields.Identifier.getSearchableFieldName());
        if ( idField == null || idField.numericValue().longValue() == record.getEventId() ) {
            break;
        }
    }

    if (record == null) {
        logger.warn("Failed to read Provenance Event for '" + d + "'. The event file may be missing or corrupted");
    }

    return record;
}
 
Example 6
@Override
protected void updateDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
  final long gen = genWriter.updateDocument(id, doc);
  // Randomly verify the udpate "took":
  if (random().nextInt(20) == 2) {
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: verify updateDocument " + id + " gen=" + gen);
    }
    nrtDeletesThread.waitForGeneration(gen);
    assertTrue(gen <= nrtDeletesThread.getSearchingGen());
    final IndexSearcher s = nrtDeletes.acquire();
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: got deletes searcher=" + s);
    }
    try {
      assertEquals("generation: " + gen, 1, s.search(new TermQuery(id), 10).totalHits.value);
    } finally {
      nrtDeletes.release(s);
    }
  }
  lastGens.set(gen);
}
 
Example 7
Source Project: lucene-solr   Source File: ICUCollationField.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example 8
Source Project: orientdb-lucene   Source File: OLuceneFacetManager.java    License: Apache License 2.0 6 votes vote down vote up
protected IndexableField buildFacetField(String f, Object val) {
  String[] path = null;
  if (val instanceof String) {

    path = ((String) val).split("/");
    // path = new String[1];
    // path[0] = (String) val;
  } else if (val instanceof Iterable) {
    Iterable iterable = (Iterable) val;
    List<String> values = new ArrayList<String>();
    for (Object s : iterable) {
      if (s instanceof String) {
        values.add((String) s);
      } else {
        throw new OIndexEngineException("Cannot facet value " + val + " because it is not a string", null);
      }
    }
    path = values.toArray(new String[values.size()]);
  }
  return new FacetField(f, path);
}
 
Example 9
private void assertData(int totalShardCount) throws IOException {
  Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
  for (int i = 0; i < totalShardCount; i++) {
    HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, ShardUtil.getShardName(i)));
    DirectoryReader reader = DirectoryReader.open(directory);
    int numDocs = reader.numDocs();
    for (int d = 0; d < numDocs; d++) {
      Document document = reader.document(d);
      IndexableField field = document.getField("id");
      Integer id = (Integer) field.numericValue();
      int partition = partitioner.getPartition(new IntWritable(id), null, totalShardCount);
      assertEquals(i, partition);
    }
    reader.close();
  }
}
 
Example 10
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example 11
Source Project: incubator-retired-blur   Source File: IndexSearcherTest.java    License: Apache License 2.0 6 votes vote down vote up
private Iterable<? extends IndexableField> getDoc(int docId, String read, String discover, String field1,
    String field2) {
  Document doc = new Document();
  doc.add(new StringField("id", Integer.toString(docId), Store.YES));
  AccessControlWriter writer = _accessControlFactory.getWriter();
  doc.add(new StringField("f1", field1, Store.YES));
  doc.add(new StringField("f2", field2, Store.YES));
  doc.add(new TextField("text", "constant text", Store.YES));
  Iterable<? extends IndexableField> fields = doc;
  if (read != null) {
    fields = writer.addReadVisiblity(read, doc);
  }
  if (discover != null) {
    fields = writer.addDiscoverVisiblity(discover, fields);
  }
  return fields;
}
 
Example 12
private static boolean isDocValue(IndexableField field) {
  if (field instanceof BinaryDocValuesField) {
    return true;
  } else if (field instanceof NumericDocValuesField) {
    return true;
  } else if (field instanceof SortedDocValuesField) {
    return true;
  } else if (field instanceof SortedSetDocValuesField) {
    return true;
  } else {
    return false;
  }
}
 
Example 13
Source Project: lucene-solr   Source File: TestBlockPostingsFormat2.java    License: Apache License 2.0 5 votes vote down vote up
/** tests terms with df % blocksize = 0 */
public void testDFBlockSizeMultiple() throws Exception {
  Document doc = newDocument();
  for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE * 16; i++) {
    for (IndexableField f : doc.getFields()) {
      ((Field) f).setStringValue(f.name() + " " + f.name() + "_2");
    }
    iw.addDocument(doc);
  }
}
 
Example 14
Source Project: localization_nifi   Source File: DocsReader.java    License: Apache License 2.0 5 votes vote down vote up
private long getByteOffset(final Document d, final RecordReader reader) {
    final IndexableField blockField = d.getField(FieldNames.BLOCK_INDEX);
    if ( blockField != null ) {
        final int blockIndex = blockField.numericValue().intValue();
        final TocReader tocReader = reader.getTocReader();
        return tocReader.getBlockOffset(blockIndex);
    }

    return d.getField(FieldNames.STORAGE_FILE_OFFSET).numericValue().longValue();
}
 
Example 15
Source Project: lucene-solr   Source File: FieldType.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Given a {@link org.apache.solr.schema.SchemaField}, create one or more {@link org.apache.lucene.index.IndexableField} instances
 * @param field the {@link org.apache.solr.schema.SchemaField}
 * @param value The value to add to the field
 * @return An array of {@link org.apache.lucene.index.IndexableField}
 *
 * @see #createField(SchemaField, Object)
 * @see #isPolyField()
 */
public List<IndexableField> createFields(SchemaField field, Object value) {
  IndexableField f = createField( field, value);
  if (field.hasDocValues() && f.fieldType().docValuesType() == null) {
    // field types that support doc values should either override createField
    // to return a field with doc values or extend createFields if this can't
    // be done in a single field instance (see StrField for example)
    throw new UnsupportedOperationException("This field type does not support doc values: " + this);
  }
  return f==null ? Collections.<IndexableField>emptyList() : Collections.singletonList(f);
}
 
Example 16
@Override
public ByteBuffer toObject(IndexableField f) {
    BytesRef bytes = f.binaryValue();
    if (bytes != null) {
        return  ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
    }
    return ByteBuffer.allocate(0);
}
 
Example 17
Source Project: lucene-solr   Source File: IndexSchema.java    License: Apache License 2.0 5 votes vote down vote up
/** Like {@link #printableUniqueKey(org.apache.lucene.document.Document)} */
public String printableUniqueKey(SolrDocument solrDoc) {
  Object val = solrDoc.getFieldValue(uniqueKeyFieldName);
  if (val == null) {
    return null;
  } else if (val instanceof IndexableField) {
    return uniqueKeyFieldType.toExternal((IndexableField) val);
  } else {
    return val.toString();
  }
}
 
Example 18
Source Project: lucene-solr   Source File: RealTimeGetComponent.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Ensure we don't have {@link org.apache.lucene.document.LazyDocument.LazyField} or equivalent.
 * It can pose problems if the searcher is about to be closed and we haven't fetched a value yet.
 */
private static IndexableField materialize(IndexableField in) {
  if (in instanceof Field) { // already materialized
    return in;
  }
  return new ClonedField(in);
}
 
Example 19
Source Project: jstarcraft-core   Source File: CollectionSortConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Iterable<IndexableField> convert(LuceneContext context, String path, Field field, LuceneSort annotatio, Type type, Object data) {
    Collection<IndexableField> indexables = new LinkedList<>();
    ParameterizedType parameterizedType = ParameterizedType.class.cast(type);
    Type[] types = parameterizedType.getActualTypeArguments();
    throw new StorageException();
}
 
Example 20
private void test(int expected, boolean rowQuery, Collection<String> discoverAuthorizations) throws IOException,
    ParseException {
  DirectoryReader reader = DirectoryReader.open(_dir);
  SuperParser parser = new SuperParser(Version.LUCENE_43, _fieldManager, rowQuery, null, ScoreType.SUPER, new Term(
      BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE));

  Query query = parser.parse("fam.string:value");

  Collection<String> readAuthorizations = null;
  Set<String> discoverableFields = new HashSet<String>();
  discoverableFields.add("rowid");
  discoverableFields.add("recordid");
  discoverableFields.add("family");
  IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations,
      discoverAuthorizations, discoverableFields, null);

  TopDocs topDocs = searcher.search(query, 10);
  assertEquals(expected, topDocs.totalHits);
  for (int i = 0; i < expected; i++) {
    int doc = topDocs.scoreDocs[i].doc;
    Document document = searcher.doc(doc);
    List<IndexableField> fields = document.getFields();
    for (IndexableField field : fields) {
      assertTrue(discoverableFields.contains(field.name()));
    }
  }
  reader.close();
}
 
Example 21
Source Project: BioSolr   Source File: NumFoundSearchComponent.java    License: Apache License 2.0 5 votes vote down vote up
private void addConvertedFieldValue(SolrDocument superDoc, Object value, SchemaField field) {
	try {
		FieldType type = field.getType();
		IndexableField indexable = type.createField(field, value, 1.0f);
		addFieldValue(superDoc, type.toObject(indexable), field);
	} catch (RuntimeException e) {
		if (! ignoreConversionErrors) {
			throw e;
		}
	}		
}
 
Example 22
Source Project: lucene-solr   Source File: FieldType.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Convert the stored-field format to an external (string, human readable)
 * value
 * @see #toInternal
 */
public String toExternal(IndexableField f) {
  // currently used in writing XML of the search result (but perhaps
  // a more efficient toXML(IndexableField f, Writer w) should be used
  // in the future.
  String val = f.stringValue();
  if (val == null) {
    // docValues will use the binary value
    val = f.binaryValue().utf8ToString();
  }
  return val;
}
 
Example 23
Source Project: jstarcraft-core   Source File: MapIndexConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Iterable<IndexableField> convert(LuceneContext context, String path, Field field, LuceneIndex annotation, Type type, Object data) {
    Collection<IndexableField> indexables = new LinkedList<>();
    // 兼容UniMi
    type = TypeUtility.refineType(type, Map.class);
    ParameterizedType parameterizedType = ParameterizedType.class.cast(type);
    Type[] types = parameterizedType.getActualTypeArguments();
    Type keyType = types[0];
    Class<?> keyClazz = TypeUtility.getRawType(keyType, null);
    Type valueType = types[1];
    Class<?> valueClazz = TypeUtility.getRawType(valueType, null);

    try {
        // TODO 此处需要代码重构
        Map<Object, Object> map = Map.class.cast(data);
        Specification keySpecification = Specification.getSpecification(keyClazz);
        IndexConverter keyConverter = context.getIndexConverter(keySpecification);
        Specification valueSpecification = Specification.getSpecification(valueClazz);
        IndexConverter valueConverter = context.getIndexConverter(valueSpecification);

        // 只索引Key,不索引Value
        return context.getIndexConverter(Specification.COLLECTION).convert(context, path, field, annotation, TypeUtility.parameterize(Collection.class, keyType), map.keySet());
    } catch (Exception exception) {
        // TODO
        throw new StorageException(exception);
    }
}
 
Example 24
Source Project: jstarcraft-core   Source File: MapStoreConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object decode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, NavigableMap<String, IndexableField> indexables) {
    String from = path;
    char character = path.charAt(path.length() - 1);
    character++;
    String to = path.substring(0, path.length() - 1) + character;
    indexables = indexables.subMap(from, true, to, false);
    Class<?> clazz = TypeUtility.getRawType(type, null);
    // 兼容UniMi
    type = TypeUtility.refineType(type, Map.class);
    ParameterizedType parameterizedType = ParameterizedType.class.cast(type);
    Type[] types = parameterizedType.getActualTypeArguments();
    Type keyType = types[0];
    Class<?> keyClazz = TypeUtility.getRawType(keyType, null);
    Type valueType = types[1];
    Class<?> valueClazz = TypeUtility.getRawType(valueType, null);

    try {
        // TODO 此处需要代码重构
        Map<Object, Object> map = (Map) context.getInstance(clazz);
        Specification keySpecification = Specification.getSpecification(keyClazz);
        StoreConverter keyConverter = context.getStoreConverter(keySpecification);
        Specification valueSpecification = Specification.getSpecification(valueClazz);
        StoreConverter valueConverter = context.getStoreConverter(valueSpecification);

        IndexableField indexable = indexables.get(path + ".size");
        int size = indexable.numericValue().intValue();
        for (int index = 0; index < size; index++) {
            Object key = keyConverter.decode(context, path + "[" + index + "_key]", field, annotation, keyType, indexables);
            Object value = valueConverter.decode(context, path + "[" + index + "_value]", field, annotation, valueType, indexables);
            map.put(key, value);
        }
        return map;
    } catch (Exception exception) {
        // TODO
        throw new StorageException(exception);
    }
}
 
Example 25
Source Project: lucene-solr   Source File: SubQueryAugmenterFactory.java    License: Apache License 2.0 5 votes vote down vote up
protected String convertFieldValue(Object val) {
  
  if (val instanceof IndexableField) {
    IndexableField f = (IndexableField)val;
    return f.stringValue();
  }
  return val.toString();
  
}
 
Example 26
Source Project: lucene-solr   Source File: RawValueTransformerFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(String name, TextWriter writer) throws IOException {
  String str = null;
  if(val instanceof IndexableField) { // delays holding it in memory
    str = ((IndexableField)val).stringValue();
  }
  else {
    str = val.toString();
  }
  writer.getWriter().write(str);
}
 
Example 27
Source Project: lucene-solr   Source File: LargeFieldTest.java    License: Apache License 2.0 5 votes vote down vote up
private void assertLazyNotLoaded(Document d, String fieldName) {
  IndexableField field = d.getField(fieldName);
  if (fieldName == BIG_FIELD) {
    assertTrue(field instanceof SolrDocumentFetcher.LargeLazyField);
    assertFalse(((SolrDocumentFetcher.LargeLazyField)field).hasBeenLoaded());
  } else {
    assertTrue(field instanceof LazyDocument.LazyField);
    assertFalse(((LazyDocument.LazyField)field).hasBeenLoaded());
  }
}
 
Example 28
Source Project: lucene-solr   Source File: TestDocument.java    License: Apache License 2.0 5 votes vote down vote up
/** test that Document.getFields() actually returns an immutable list */
public void testGetFieldsImmutable() {
  Document doc = makeDocumentWithFields();
  assertEquals(10, doc.getFields().size());
  List<IndexableField> fields = doc.getFields();
  expectThrows(UnsupportedOperationException.class, () -> {
    fields.add(new StringField("name", "value", Field.Store.NO));
  });

  expectThrows(UnsupportedOperationException.class, () -> {
    fields.clear();
  });
}
 
Example 29
Source Project: crate   Source File: FieldMapper.java    License: Apache License 2.0 5 votes vote down vote up
protected void createFieldNamesField(ParseContext context, List<IndexableField> fields) {
    FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context.docMapper()
            .metadataMapper(FieldNamesFieldMapper.class).fieldType();
    if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) {
        for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) {
            fields.add(new Field(FieldNamesFieldMapper.NAME, fieldName, fieldNamesFieldType));
        }
    }
}
 
Example 30
Source Project: lucene-solr   Source File: SortableTextField.java    License: Apache License 2.0 5 votes vote down vote up
private static List<IndexableField> getIndexableFields(SchemaField field, IndexableField f, BytesRef bytes) {
  final IndexableField docval = field.multiValued()
    ? new SortedSetDocValuesField(field.getName(), bytes)
    : new SortedDocValuesField(field.getName(), bytes);
  
  if (null == f) {
    return Collections.singletonList(docval);
  } 
  return Arrays.asList(f, docval);
}