Java Code Examples for org.apache.solr.schema.SchemaField#stored()

The following examples show how to use org.apache.solr.schema.SchemaField#stored() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SolrDocumentFetcher.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a collection of the names of all stored fields which can be highlighted the index reader knows about.
 */
public Collection<String> getStoredHighlightFieldNames() {
  synchronized (this) {
    if (storedHighlightFieldNames == null) {
      storedHighlightFieldNames = new LinkedList<>();
      for (FieldInfo fieldInfo : searcher.getFieldInfos()) {
        final String fieldName = fieldInfo.name;
        try {
          SchemaField field = searcher.getSchema().getField(fieldName);
          if (field.stored() && ((field.getType() instanceof org.apache.solr.schema.TextField)
              || (field.getType() instanceof org.apache.solr.schema.StrField))) {
            storedHighlightFieldNames.add(fieldName);
          }
        } catch (RuntimeException e) { // getField() throws a SolrException, but it arrives as a RuntimeException
          log.warn("Field [{}] found in index, but not defined in schema.", fieldName);
        }
      }
    }
    return storedHighlightFieldNames;
  }
}
 
Example 2
Source File: RealTimeGetComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static SolrInputDocument toSolrInputDocument(Document doc, IndexSchema schema) {
  SolrInputDocument out = new SolrInputDocument();
  for( IndexableField f : doc.getFields() ) {
    String fname = f.name();
    SchemaField sf = schema.getFieldOrNull(f.name());
    Object val = null;
    if (sf != null) {
      if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue;
      val = sf.getType().toObject(f);   // object or external string?
    } else {
      val = f.stringValue();
      if (val == null) val = f.numericValue();
      if (val == null) val = f.binaryValue();
      if (val == null) val = f;
    }

    // todo: how to handle targets of copy fields (including polyfield sub-fields)?
    out.addField(fname, val);
  }
  return out;
}
 
Example 3
Source File: TestRetrieveFieldsOptimizer.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
IndexSchema addFields(IndexSchema schema) {

    List<SchemaField> fieldsToAdd = new ArrayList<>();

    for (RetrieveField field : fields.values()) {
      allFields.add(field);
      SchemaField schemaField = field.schemaField;
      fieldsToAdd.add(schemaField);
      if (schemaField.multiValued()) {
        multiValuedFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored() == false) {
        dvNotStoredFields.add(field);
      }
      if (schemaField.hasDocValues() == false && schemaField.stored()) {
        storedNotDvFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored()) {
        storedAndDvFields.add(field);
      }
      if (schemaField.stored() && schemaField.multiValued()) {
        storedMvFields.add(field);
      }
    }
    return schema.addFields(fieldsToAdd, Collections.emptyMap(), false);
  }
 
Example 4
Source File: QueryAutoFilteringComponent.java    From query-autofiltering-component with Apache License 2.0 6 votes vote down vote up
private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      SchemaField field = schema.getField(fieldName);
      if (field.stored() && field.getType() instanceof StrField ) {
        strFields.add( fieldName );
      }
    }
  }
    
  return strFields;
}
 
Example 5
Source File: QueryAutoFilteringComponent.java    From query-autofiltering-component with Apache License 2.0 6 votes vote down vote up
private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      try {
        SchemaField field = schema.getField(fieldName);
        if (field.stored() && field.getType() instanceof StrField ) {
          strFields.add( fieldName );
        }
      }
      catch (Throwable e )
      {
          
      }
    }
  }
    
  return strFields;
}
 
Example 6
Source File: TabularResponseWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Returns true if field needs to be skipped else false
 * @param field name of the field
 * @return boolean value
 */
public boolean shouldSkipField(String field) {
  Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
  SchemaField sf = schema.getFieldOrNull(field);

  // Return stored fields or useDocValuesAsStored=true fields,
  // unless an explicit field list is specified
  return  (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
      sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()));
}
 
Example 7
Source File: VersionInfo.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the specified
 * schema, after verifying that it is indexed, stored, and single-valued.  
 * If any of these pre-conditions are not met, it throws a SolrException 
 * with a user suitable message indicating the problem.
 */
public static SchemaField getAndCheckVersionField(IndexSchema schema) 
  throws SolrException {
  final String errPrefix = VERSION_FIELD + " field must exist in schema and be searchable (indexed or docValues) and retrievable(stored or docValues) and not multiValued";
  SchemaField sf = schema.getFieldOrNull(VERSION_FIELD);

  if (null == sf) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " does not exist)");
  }
  if ( !sf.indexed() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not searchable");
  }
  if ( !sf.stored() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not retrievable");
  }
  if ( sf.multiValued() ) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " is multiValued");
  }
  
  return sf;
}
 
Example 8
Source File: DocBasedVersionConstraintsProcessorFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void inform(SolrCore core) {

  if (core.getUpdateHandler().getUpdateLog() == null) {
    throw new SolrException(SERVER_ERROR,
        "updateLog must be enabled.");
  }

  if (core.getLatestSchema().getUniqueKeyField() == null) {
    throw new SolrException(SERVER_ERROR,
        "schema must have uniqueKey defined.");
  }

  useFieldCache = true;
  for (String versionField : versionFields) {
    SchemaField userVersionField = core.getLatestSchema().getField(versionField);
    if (userVersionField == null || !userVersionField.stored() || userVersionField.multiValued()) {
      throw new SolrException(SERVER_ERROR,
          "field " + versionField + " must be defined in schema, be stored, and be single valued.");
    }
    if (useFieldCache) {
      try {
        userVersionField.getType().getValueSource(userVersionField, null);
      } catch (Exception e) {
        useFieldCache = false;
        log.warn("Can't use fieldcache/valuesource: {}", e.getMessage());
      }
    }
  }
  
  canCreateTombstoneDocument(core.getLatestSchema());
}
 
Example 9
Source File: SolrDocumentFetcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) {
  if (!schemaField.hasDocValues() || !schemaField.stored()) return false;
  if (schemaField.multiValued()) return false;
  DocValuesType docValuesType = fieldInfo.getDocValuesType();
  NumberType numberType = schemaField.getType().getNumberType();
  // can not decode a numeric without knowing its numberType
  if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) {
    return false;
  }
  return true;
}
 
Example 10
Source File: RealTimeGetComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static SolrInputDocument toSolrInputDocument(SolrDocument doc, IndexSchema schema) {
  SolrInputDocument out = new SolrInputDocument();
  for( String fname : doc.getFieldNames() ) {
    boolean fieldArrayListCreated = false;
    SchemaField sf = schema.getFieldOrNull(fname);
    if (sf != null) {
      if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue;
    }
    for (Object val: doc.getFieldValues(fname)) {
      if (val instanceof Field) {
        Field f = (Field) val;
        if (sf != null) {
          val = sf.getType().toObject(f);   // object or external string?
        } else {
          val = f.stringValue();
          if (val == null) val = f.numericValue();
          if (val == null) val = f.binaryValue();
          if (val == null) val = f;
        }
      } else if(val instanceof SolrDocument) {
        val = toSolrInputDocument((SolrDocument) val, schema);
        if(!fieldArrayListCreated && doc.getFieldValue(fname) instanceof Collection) {
          // previous value was array so we must return as an array even if was a single value array
          out.setField(fname, Lists.newArrayList(val));
          fieldArrayListCreated = true;
          continue;
        }
      }
      out.addField(fname, val);
    }
  }
  return out;
}
 
Example 11
Source File: BinaryDocValuesField.java    From liresolr with GNU General Public License v2.0 5 votes vote down vote up
@Override
    public IndexableField createField(SchemaField field, Object val /*, float boost*/) {
        if (val == null) return null;
        if (!field.stored()) {
            return null;
        }
        byte[] buf = null;
        int offset = 0, len = 0;
        if (val instanceof byte[]) {
            buf = (byte[]) val;
            len = buf.length;
        } else if (val instanceof ByteBuffer && ((ByteBuffer)val).hasArray()) {
            ByteBuffer byteBuf = (ByteBuffer) val;
            buf = byteBuf.array();
            offset = byteBuf.position();
            len = byteBuf.limit() - byteBuf.position();
        } else {
            String strVal = val.toString();
            //the string has to be a base64 encoded string
            buf = Base64.base64ToByteArray(strVal);
            offset = 0;
            len = buf.length;
        }

        Field f = new org.apache.lucene.document.BinaryDocValuesField(field.getName(), new BytesRef(buf, offset, len));
//        Field f = new org.apache.lucene.document.StoredField(field.getName(), buf, offset, len);
        //f.setBoost(boost);
        return f;
    }
 
Example 12
Source File: SolrDocumentFetcher.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({"unchecked"})
SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) {
  this.searcher = searcher;
  this.enableLazyFieldLoading = solrConfig.enableLazyFieldLoading;
  if (cachingEnabled) {
    documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
  } else {
    documentCache = null;
  }

  final Set<String> nonStoredDVsUsedAsStored = new HashSet<>();
  final Set<String> allNonStoredDVs = new HashSet<>();
  final Set<String> nonStoredDVsWithoutCopyTargets = new HashSet<>();
  final Set<String> storedLargeFields = new HashSet<>();
  final Set<String> dvsCanSubstituteStored = new HashSet<>();
  final Set<String> allStoreds = new HashSet<>();

  for (FieldInfo fieldInfo : searcher.getFieldInfos()) { // can find materialized dynamic fields, unlike using the Solr IndexSchema.
    final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldInfo.name);
    if (schemaField == null) {
      continue;
    }
    if (canSubstituteDvForStored(fieldInfo, schemaField)) {
      dvsCanSubstituteStored.add(fieldInfo.name);
    }
    if (schemaField.stored()) {
      allStoreds.add(fieldInfo.name);
    }
    if (!schemaField.stored() && schemaField.hasDocValues()) {
      if (schemaField.useDocValuesAsStored()) {
        nonStoredDVsUsedAsStored.add(fieldInfo.name);
      }
      allNonStoredDVs.add(fieldInfo.name);
      if (!searcher.getSchema().isCopyFieldTarget(schemaField)) {
        nonStoredDVsWithoutCopyTargets.add(fieldInfo.name);
      }
    }
    if (schemaField.stored() && schemaField.isLarge()) {
      storedLargeFields.add(schemaField.getName());
    }
  }

  this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored);
  this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
  this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets);
  this.largeFields = Collections.unmodifiableSet(storedLargeFields);
  this.dvsCanSubstituteStored = Collections.unmodifiableSet(dvsCanSubstituteStored);
  this.allStored = Collections.unmodifiableSet(allStoreds);
}
 
Example 13
Source File: RealTimeGetComponent.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/**
 * Converts a SolrInputDocument to SolrDocument, using an IndexSchema instance.
 *
 * @param sdoc The input document.
 * @param schema The index schema.
 * @param forInPlaceUpdate Whether the document is being used for an in place update,
 *                         see {@link DocumentBuilder#toDocument(SolrInputDocument, IndexSchema, boolean, boolean)}
 */
public static SolrDocument toSolrDoc(SolrInputDocument sdoc, IndexSchema schema, boolean forInPlaceUpdate) {
  // TODO what about child / nested docs?
  // TODO: do something more performant than this double conversion
  Document doc = DocumentBuilder.toDocument(sdoc, schema, forInPlaceUpdate, true);

  // copy the stored fields only
  Document out = new Document();
  for (IndexableField f : doc.getFields()) {
    if (f.fieldType().stored()) {
      out.add(f);
    } else if (f.fieldType().docValuesType() != DocValuesType.NONE) {
      SchemaField schemaField = schema.getFieldOrNull(f.name());
      if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored()) {
        out.add(f);
      }
    } else {
      log.debug("Don't know how to handle field {}", f);
    }
  }

  SolrDocument solrDoc = toSolrDoc(out, schema);

  // add child docs
  for(SolrInputField solrInputField: sdoc) {
    if(solrInputField.getFirstValue() instanceof SolrInputDocument) {
      // is child doc
      Object val = solrInputField.getValue();
      Iterator<SolrDocument> childDocs = solrInputField.getValues().stream()
          .map(x -> toSolrDoc((SolrInputDocument) x, schema)).iterator();
      if(val instanceof Collection) {
        // add as collection even if single element collection
        solrDoc.setField(solrInputField.getName(), Lists.newArrayList(childDocs));
      } else {
        // single child doc
        solrDoc.setField(solrInputField.getName(), childDocs.next());
      }
    }
  }

  return solrDoc;
}
 
Example 14
Source File: AtomicUpdateDocumentMerger.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/**
 * Given a schema field, return whether or not such a field is supported for an in-place update.
 * Note: If an update command has updates to only supported fields (and _version_ is also supported),
 * only then is such an update command executed as an in-place update.
 */
public static boolean isSupportedFieldForInPlaceUpdate(SchemaField schemaField) {
  return !(schemaField.indexed() || schemaField.stored() || !schemaField.hasDocValues() || 
      schemaField.multiValued() || !(schemaField.getType() instanceof NumericValueFieldType));
}