org.apache.lucene.index.IndexableField Java Examples

The following examples show how to use org.apache.lucene.index.IndexableField. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OLuceneFacetManager.java    From orientdb-lucene with Apache License 2.0 6 votes vote down vote up
protected IndexableField buildFacetField(String f, Object val) {
  String[] path = null;
  if (val instanceof String) {

    path = ((String) val).split("/");
    // path = new String[1];
    // path[0] = (String) val;
  } else if (val instanceof Iterable) {
    Iterable iterable = (Iterable) val;
    List<String> values = new ArrayList<String>();
    for (Object s : iterable) {
      if (s instanceof String) {
        values.add((String) s);
      } else {
        throw new OIndexEngineException("Cannot facet value " + val + " because it is not a string", null);
      }
    }
    path = values.toArray(new String[values.size()]);
  }
  return new FacetField(f, path);
}
 
Example #2
Source File: ArrayStoreConverter.java    From jstarcraft-core with Apache License 2.0 6 votes vote down vote up
@Override
public NavigableMap<String, IndexableField> encode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, Object instance) {
    NavigableMap<String, IndexableField> indexables = new TreeMap<>();
    Class<?> componentClass = null;
    Type componentType = null;
    if (type instanceof GenericArrayType) {
        GenericArrayType genericArrayType = GenericArrayType.class.cast(type);
        componentType = genericArrayType.getGenericComponentType();
        componentClass = TypeUtility.getRawType(componentType, null);
    } else {
        Class<?> clazz = TypeUtility.getRawType(type, null);
        componentType = clazz.getComponentType();
        componentClass = clazz.getComponentType();
    }
    Specification specification = Specification.getSpecification(componentClass);
    StoreConverter converter = context.getStoreConverter(specification);
    int size = Array.getLength(instance);
    IndexableField indexable = new StoredField(path + ".size", size);
    indexables.put(path + ".size", indexable);
    for (int index = 0; index < size; index++) {
        Object element = Array.get(instance, index);
        indexables.putAll(converter.encode(context, path + "[" + index + "]", field, annotation, componentType, element));
    }
    return indexables;
}
 
Example #3
Source File: LazyDocument.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a StorableField whose value will be lazy loaded if and 
 * when it is used. 
 * <p>
 * <b>NOTE:</b> This method must be called once for each value of the field 
 * name specified in sequence that the values exist.  This method may not be 
 * used to generate multiple, lazy, StorableField instances referring to 
 * the same underlying StorableField instance.
 * </p>
 * <p>
 * The lazy loading of field values from all instances of StorableField 
 * objects returned by this method are all backed by a single StoredDocument 
 * per LazyDocument instance.
 * </p>
 */
public IndexableField getField(FieldInfo fieldInfo) {  

  fieldNames.add(fieldInfo.name);
  List<LazyField> values = fields.get(fieldInfo.number);
  if (null == values) {
    values = new ArrayList<>();
    fields.put(fieldInfo.number, values);
  } 

  LazyField value = new LazyField(fieldInfo.name, fieldInfo.number);
  values.add(value);

  synchronized (this) {
    // edge case: if someone asks this LazyDoc for more LazyFields
    // after other LazyFields from the same LazyDoc have been
    // actuallized, we need to force the doc to be re-fetched
    // so the new LazyFields are also populated.
    doc = null;
  }
  return value;
}
 
Example #4
Source File: DocsReader.java    From nifi with Apache License 2.0 6 votes vote down vote up
private ProvenanceEventRecord getRecord(final Document d, final RecordReader reader) throws IOException {
    final IndexableField blockField = d.getField(FieldNames.BLOCK_INDEX);
    if ( blockField == null ) {
        reader.skipTo(getByteOffset(d, reader));
    } else {
        reader.skipToBlock(blockField.numericValue().intValue());
    }

    StandardProvenanceEventRecord record;
    while ( (record = reader.nextRecord()) != null) {
        final IndexableField idField = d.getField(SearchableFields.Identifier.getSearchableFieldName());
        if ( idField == null || idField.numericValue().longValue() == record.getEventId() ) {
            break;
        }
    }

    if (record == null) {
        logger.warn("Failed to read Provenance Event for '" + d + "'. The event file may be missing or corrupted");
    }

    return record;
}
 
Example #5
Source File: TestControlledRealTimeReopenThread.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected void updateDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
  final long gen = genWriter.updateDocument(id, doc);
  // Randomly verify the udpate "took":
  if (random().nextInt(20) == 2) {
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: verify updateDocument " + id + " gen=" + gen);
    }
    nrtDeletesThread.waitForGeneration(gen);
    assertTrue(gen <= nrtDeletesThread.getSearchingGen());
    final IndexSearcher s = nrtDeletes.acquire();
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: got deletes searcher=" + s);
    }
    try {
      assertEquals("generation: " + gen, 1, s.search(new TermQuery(id), 10).totalHits.value);
    } finally {
      nrtDeletes.release(s);
    }
  }
  lastGens.set(gen);
}
 
Example #6
Source File: TableShardCountCollapserTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private void assertData(int totalShardCount) throws IOException {
  Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
  for (int i = 0; i < totalShardCount; i++) {
    HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, ShardUtil.getShardName(i)));
    DirectoryReader reader = DirectoryReader.open(directory);
    int numDocs = reader.numDocs();
    for (int d = 0; d < numDocs; d++) {
      Document document = reader.document(d);
      IndexableField field = document.getField("id");
      Integer id = (Integer) field.numericValue();
      int partition = partitioner.getPartition(new IntWritable(id), null, totalShardCount);
      assertEquals(i, partition);
    }
    reader.close();
  }
}
 
Example #7
Source File: CurrencyFieldType.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object externalVal) {
  CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);

  List<IndexableField> f = new ArrayList<>();
  SchemaField amountField = getAmountField(field);
  f.add(amountField.createField(String.valueOf(value.getAmount())));
  SchemaField currencyField = getCurrencyField(field);
  f.add(currencyField.createField(value.getCurrencyCode()));

  if (field.stored()) {
    String storedValue = externalVal.toString().trim();
    if (storedValue.indexOf(",") < 0) {
      storedValue += "," + defaultCurrency;
    }
    f.add(createField(field.getName(), storedValue, StoredField.TYPE));
  }

  return f;
}
 
Example #8
Source File: ICUCollationField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example #9
Source File: TestSortableTextField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testWhiteboxCreateFields() throws Exception {
  List<IndexableField> values = null;

  // common case...
  for (String field : Arrays.asList("keyword_stxt", "keyword_dv_stxt",
                                    "whitespace_stxt", "whitespace_f_stxt", "whitespace_l_stxt")) {
    values = createIndexableFields(field);
    assertEquals(field, 2, values.size());
    assertThat(field, values.get(0), instanceOf(Field.class));
    assertThat(field, values.get(1), instanceOf(SortedDocValuesField.class));
  }
  
  // special cases...
  values = createIndexableFields("whitespace_nois_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(SortedDocValuesField.class));
  //
  values = createIndexableFields("whitespace_nodv_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  //
  values = createIndexableFields("whitespace_m_stxt");
  assertEquals(2, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  assertThat(values.get(1), instanceOf(SortedSetDocValuesField.class));      
}
 
Example #10
Source File: SecureAtomicReaderTestBase.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example #11
Source File: IndexSearcherTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private Iterable<? extends IndexableField> getDoc(int docId, String read, String discover, String field1,
    String field2) {
  Document doc = new Document();
  doc.add(new StringField("id", Integer.toString(docId), Store.YES));
  AccessControlWriter writer = _accessControlFactory.getWriter();
  doc.add(new StringField("f1", field1, Store.YES));
  doc.add(new StringField("f2", field2, Store.YES));
  doc.add(new TextField("text", "constant text", Store.YES));
  Iterable<? extends IndexableField> fields = doc;
  if (read != null) {
    fields = writer.addReadVisiblity(read, doc);
  }
  if (discover != null) {
    fields = writer.addDiscoverVisiblity(discover, fields);
  }
  return fields;
}
 
Example #12
Source File: FilterAccessControlFactory.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static FieldType getFieldTypeNotStored(IndexableField indexableField) {
  Field field = (Field) indexableField;
  FieldType fieldType = field.fieldType();
  FieldType result = new FieldType(fieldType);
  result.setStored(false);
  result.freeze();
  return result;
}
 
Example #13
Source File: RealTimeGetComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Ensure we don't have {@link org.apache.lucene.document.LazyDocument.LazyField} or equivalent.
 * It can pose problems if the searcher is about to be closed and we haven't fetched a value yet.
 */
private static IndexableField materialize(IndexableField in) {
  if (in instanceof Field) { // already materialized
    return in;
  }
  return new ClonedField(in);
}
 
Example #14
Source File: SubQueryAugmenterFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected String convertFieldValue(Object val) {
  
  if (val instanceof IndexableField) {
    IndexableField f = (IndexableField)val;
    return f.stringValue();
  }
  return val.toString();
  
}
 
Example #15
Source File: AbstractLuceneQueryVisitorTest.java    From cxf with Apache License 2.0 5 votes vote down vote up
protected void doTestIntContentMatchWithQuery(Query query) throws Exception {

        ScoreDoc[] hits = isearcher.search(query, 1000).scoreDocs;
        assertEquals(1, hits.length);
        // Iterate through the results:
        for (int i = 0; i < hits.length; i++) {
            Document hitDoc = isearcher.doc(hits[i].doc);
            IndexableField field = hitDoc.getField("intfield");
            assertEquals(4, field.numericValue().intValue());
        }

    }
 
Example #16
Source File: RawValueTransformerFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void write(String name, TextWriter writer) throws IOException {
  String str = null;
  if(val instanceof IndexableField) { // delays holding it in memory
    str = ((IndexableField)val).stringValue();
  }
  else {
    str = val.toString();
  }
  writer.getWriter().write(str);
}
 
Example #17
Source File: IndexSchema.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Like {@link #printableUniqueKey(org.apache.lucene.document.Document)} */
public String printableUniqueKey(SolrDocument solrDoc) {
  Object val = solrDoc.getFieldValue(uniqueKeyFieldName);
  if (val == null) {
    return null;
  } else if (val instanceof IndexableField) {
    return uniqueKeyFieldType.toExternal((IndexableField) val);
  } else {
    return val.toString();
  }
}
 
Example #18
Source File: SolrInformationServer.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
private String getFieldValueString(SolrDocument doc, String fieldName)
{
    IndexableField field = (IndexableField)doc.getFieldValue(fieldName);
    String value = null;
    if (field != null)
    {
        value = field.stringValue();
    }
    return value;
}
 
Example #19
Source File: MapStoreConverter.java    From jstarcraft-core with Apache License 2.0 5 votes vote down vote up
@Override
public Object decode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, NavigableMap<String, IndexableField> indexables) {
    String from = path;
    char character = path.charAt(path.length() - 1);
    character++;
    String to = path.substring(0, path.length() - 1) + character;
    indexables = indexables.subMap(from, true, to, false);
    Class<?> clazz = TypeUtility.getRawType(type, null);
    // 兼容UniMi
    type = TypeUtility.refineType(type, Map.class);
    ParameterizedType parameterizedType = ParameterizedType.class.cast(type);
    Type[] types = parameterizedType.getActualTypeArguments();
    Type keyType = types[0];
    Class<?> keyClazz = TypeUtility.getRawType(keyType, null);
    Type valueType = types[1];
    Class<?> valueClazz = TypeUtility.getRawType(valueType, null);

    try {
        // TODO 此处需要代码重构
        Map<Object, Object> map = (Map) context.getInstance(clazz);
        Specification keySpecification = Specification.getSpecification(keyClazz);
        StoreConverter keyConverter = context.getStoreConverter(keySpecification);
        Specification valueSpecification = Specification.getSpecification(valueClazz);
        StoreConverter valueConverter = context.getStoreConverter(valueSpecification);

        IndexableField indexable = indexables.get(path + ".size");
        int size = indexable.numericValue().intValue();
        for (int index = 0; index < size; index++) {
            Object key = keyConverter.decode(context, path + "[" + index + "_key]", field, annotation, keyType, indexables);
            Object value = valueConverter.decode(context, path + "[" + index + "_value]", field, annotation, valueType, indexables);
            map.put(key, value);
        }
        return map;
    } catch (Exception exception) {
        // TODO
        throw new StorageException(exception);
    }
}
 
Example #20
Source File: LargeFieldTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void assertLazyNotLoaded(Document d, String fieldName) {
  IndexableField field = d.getField(fieldName);
  if (fieldName == BIG_FIELD) {
    assertTrue(field instanceof SolrDocumentFetcher.LargeLazyField);
    assertFalse(((SolrDocumentFetcher.LargeLazyField)field).hasBeenLoaded());
  } else {
    assertTrue(field instanceof LazyDocument.LazyField);
    assertFalse(((LazyDocument.LazyField)field).hasBeenLoaded());
  }
}
 
Example #21
Source File: FilterAccessControlFactory.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static boolean isDocValue(IndexableField field) {
  if (field instanceof BinaryDocValuesField) {
    return true;
  } else if (field instanceof NumericDocValuesField) {
    return true;
  } else if (field instanceof SortedDocValuesField) {
    return true;
  } else if (field instanceof SortedSetDocValuesField) {
    return true;
  } else {
    return false;
  }
}
 
Example #22
Source File: TestDocument.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** test that Document.getFields() actually returns an immutable list */
public void testGetFieldsImmutable() {
  Document doc = makeDocumentWithFields();
  assertEquals(10, doc.getFields().size());
  List<IndexableField> fields = doc.getFields();
  expectThrows(UnsupportedOperationException.class, () -> {
    fields.add(new StringField("name", "value", Field.Store.NO));
  });

  expectThrows(UnsupportedOperationException.class, () -> {
    fields.clear();
  });
}
 
Example #23
Source File: FieldMapper.java    From crate with Apache License 2.0 5 votes vote down vote up
protected void createFieldNamesField(ParseContext context, List<IndexableField> fields) {
    FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context.docMapper()
            .metadataMapper(FieldNamesFieldMapper.class).fieldType();
    if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) {
        for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) {
            fields.add(new Field(FieldNamesFieldMapper.NAME, fieldName, fieldNamesFieldType));
        }
    }
}
 
Example #24
Source File: FilterAccessControlFactory.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private static Set<String> getFieldsToMask(Iterable<? extends IndexableField> fields) {
  Set<String> result = new HashSet<String>();
  for (IndexableField field : fields) {
    if (field.name().equals(READ_MASK_FIELD)) {
      result.add(getFieldNameOnly(field.stringValue()));
    }
  }
  return result;
}
 
Example #25
Source File: SolrInformationServer.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
@Override
public Set<Long> getErrorDocIds() throws IOException
{
    Set<Long> errorDocIds = new HashSet<>();
    RefCounted<SolrIndexSearcher> refCounted = null;
    try
    {
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        TermQuery errorQuery = new TermQuery(new Term(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE));
        DocListCollector docListCollector = new DocListCollector();
        searcher.search(errorQuery, docListCollector);
        IntArrayList docList = docListCollector.getDocs();
        int size = docList.size();

        for (int i = 0; i < size; ++i)
        {
            int doc = docList.get(i);
            Document document = searcher.doc(doc, REQUEST_ONLY_ID_FIELD);
            IndexableField id = document.getField(FIELD_SOLR4_ID);
            String idString = id.stringValue();

            if (idString.startsWith(PREFIX_ERROR))
            {
                idString = idString.substring(PREFIX_ERROR.length());
            }

            errorDocIds.add(Long.valueOf(idString));
        }
    }
    finally
    {
        ofNullable(refCounted).ifPresent(RefCounted::decref);
    }
    return errorDocIds;
}
 
Example #26
Source File: DatePointField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public IndexableField createField(SchemaField field, Object value) {
  Date date = (value instanceof Date)
      ? ((Date)value)
      : DateMathParser.parseMath(null, value.toString());
  return new LongPoint(field.getName(), date.getTime());
}
 
Example #27
Source File: ParseContext.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public BytesRef getBinaryValue(String name) {
    for (IndexableField f : fields) {
        if (f.name().equals(name) && f.binaryValue() != null) {
            return f.binaryValue();
        }
    }
    return null;
}
 
Example #28
Source File: SortableTextField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static List<IndexableField> getIndexableFields(SchemaField field, IndexableField f, BytesRef bytes) {
  final IndexableField docval = field.multiValued()
    ? new SortedSetDocValuesField(field.getName(), bytes)
    : new SortedDocValuesField(field.getName(), bytes);
  
  if (null == f) {
    return Collections.singletonList(docval);
  } 
  return Arrays.asList(f, docval);
}
 
Example #29
Source File: AbstractSpatialFieldType.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object val) {
  String shapeStr = null;
  Shape shape;
  if (val instanceof Shape) {
    shape = ((Shape) val);
  } else {
    shapeStr = val.toString();
    shape = parseShape(shapeStr);
  }
  if (shape == null) {
    log.debug("Field {}: null shape for input: {}", field, val);
    return Collections.emptyList();
  }

  List<IndexableField> result = new ArrayList<>();
  if (field.indexed() || field.hasDocValues()) {
    T strategy = getStrategy(field.getName());
    result.addAll(Arrays.asList(strategy.createIndexableFields(shape)));
  }

  if (field.stored()) {
    result.add(new StoredField(field.getName(), getStoredValue(shape, shapeStr)));
  }

  return result;
}
 
Example #30
Source File: ParseContext.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public String get(String name) {
    for (IndexableField f : fields) {
        if (f.name().equals(name) && f.stringValue() != null) {
            return f.stringValue();
        }
    }
    return null;
}