Java Code Examples for org.apache.solr.schema.SchemaField#indexed()

@Override
public void inform(SolrCore core) {
  final SchemaField field = core.getLatestSchema().getFieldOrNull(getSignatureField());
  if (null == field) {
    throw new SolrException
      (ErrorCode.SERVER_ERROR,
       "Can't use signatureField which does not exist in schema: "
       + getSignatureField());
  }

  if (getOverwriteDupes() && ( ! field.indexed() ) ) {
    throw new SolrException
      (ErrorCode.SERVER_ERROR,
       "Can't set overwriteDupes when signatureField is not indexed: "
       + getSignatureField());
  }
}
 

/**
 * Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the specified
 * schema, after verifying that it is indexed, stored, and single-valued.  
 * If any of these pre-conditions are not met, it throws a SolrException 
 * with a user suitable message indicating the problem.
 */
public static SchemaField getAndCheckVersionField(IndexSchema schema) 
  throws SolrException {
  final String errPrefix = VERSION_FIELD + " field must exist in schema and be searchable (indexed or docValues) and retrievable(stored or docValues) and not multiValued";
  SchemaField sf = schema.getFieldOrNull(VERSION_FIELD);

  if (null == sf) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " does not exist)");
  }
  if ( !sf.indexed() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not searchable");
  }
  if ( !sf.stored() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not retrievable");
  }
  if ( sf.multiValued() ) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " is multiValued");
  }
  
  return sf;
}
 

@Override
public Query getResultQuery(SchemaField matchField, boolean useAutomaton) {
  if (collectorTerms == null || collectorTerms.size() == 0) {
    // return null if there are no terms (edges) to traverse.
    return null;
  } else {
    // Create a query
    Query q = null;

    // TODO: see if we should dynamically select this based on the frontier size.
    if (useAutomaton) {
      // build an automaton based query for the frontier.
      Automaton autn = buildAutomaton(collectorTerms);
      AutomatonQuery autnQuery = new AutomatonQuery(new Term(matchField.getName()), autn);
      q = autnQuery;
    } else {
      List<BytesRef> termList = new ArrayList<>(collectorTerms.size());
      for (int i = 0; i < collectorTerms.size(); i++) {
        BytesRef ref = new BytesRef();
        collectorTerms.get(i, ref);
        termList.add(ref);
      }
      q = (matchField.hasDocValues() && !matchField.indexed())
              ? new DocValuesTermsQuery(matchField.getName(), termList)
              : new TermInSetQuery(matchField.getName(), termList);
    }

    return q;
  }
}
 

@Test
public void testSyntax() throws Exception {
  // a bare * should be treated as *:*
  assertJQ(req("q", "*", "df", "doesnotexist_s")
      , "/response/docs/[0]=="   // make sure we get something...
  );
  assertJQ(req("q", "doesnotexist_s:*")
      , "/response/numFound==0"   // nothing should be found
  );
  assertJQ(req("q", "doesnotexist_s:( * * * )")
      , "/response/numFound==0"   // nothing should be found
  );

  // length of date math caused issues...
  {
    SchemaField foo_dt = h.getCore().getLatestSchema().getField("foo_dt");
    String expected = "foo_dt:2013-09-11T00:00:00Z";
    if (foo_dt.getType().isPointField()) {
      expected = "(foo_dt:[1378857600000 TO 1378857600000])";
      if (foo_dt.hasDocValues() && foo_dt.indexed()) {
        expected = "IndexOrDocValuesQuery"+expected ;
      }
    }
    assertJQ(req("q", "foo_dt:\"2013-03-08T00:46:15Z/DAY+000MILLISECONDS+00SECONDS+00MINUTES+00HOURS+0000000000YEARS+6MONTHS+3DAYS\"", "debug", "query")
             , "/debug/parsedquery=='"+expected+"'");
  }
}
 

private static SimpleOrderedMap<Object> getIndexedFieldsInfo(
		SolrQueryRequest req) throws Exception {

	SolrIndexSearcher searcher = req.getSearcher();
	SolrParams params = req.getParams();

	Set<String> fields = null;
	String fl = params.get(CommonParams.FL);
	if (fl != null) {
		fields = new TreeSet<>(Arrays.asList(fl.split("[,\\s]+")));
	}

	LeafReader reader = searcher.getSlowAtomicReader();
	IndexSchema schema = searcher.getSchema();

	// Don't be tempted to put this in the loop below, the whole point here
	// is to alphabetize the fields!
	Set<String> fieldNames = new TreeSet<>();
	for (FieldInfo fieldInfo : reader.getFieldInfos()) {
		fieldNames.add(fieldInfo.name);
	}

	// Walk the term enum and keep a priority queue for each map in our set
	SimpleOrderedMap<Object> vInfo = new SimpleOrderedMap<>();
	SimpleOrderedMap<Object> aInfo = new SimpleOrderedMap<>();

	for (String fieldName : fieldNames) {
		if (fields != null && !fields.contains(fieldName)
				&& !fields.contains("*")) {
			continue; // we're not interested in this field Still an issue
						// here
		}

		SimpleOrderedMap<Object> fieldMap = new SimpleOrderedMap<>();

		SchemaField sfield = schema.getFieldOrNull(fieldName);
		FieldType ftype = (sfield == null) ? null : sfield.getType();

		fieldMap.add("type", (ftype == null) ? null : ftype.getTypeName());
		fieldMap.add("schema", getFieldFlags(sfield));
		if (sfield != null && schema.isDynamicField(sfield.getName())
				&& schema.getDynamicPattern(sfield.getName()) != null) {
			fieldMap.add("dynamicBase",
					schema.getDynamicPattern(sfield.getName()));
		}
		Terms terms = reader.fields().terms(fieldName);
		if (terms == null) { // Not indexed, so we need to report what we
								// can (it made it through the fl param if
								// specified)
			vInfo.add(AlfrescoSolrDataModel.getInstance()
					.getAlfrescoPropertyFromSchemaField(fieldName),
					fieldMap);
			aInfo.add(fieldName, fieldMap);
			continue;
		}

		if (sfield != null && sfield.indexed()) {
			if (params.getBool(INCLUDE_INDEX_FIELD_FLAGS, true)) {
				Document doc = getFirstLiveDoc(terms, reader);

				if (doc != null) {
					// Found a document with this field
					try {
						IndexableField fld = doc.getField(fieldName);
						if (fld != null) {
							fieldMap.add("index", getFieldFlags(fld));
						} else {
							// it is a non-stored field...
							fieldMap.add("index", "(unstored field)");
						}
					} catch (Exception ex) {
						log.warn("error reading field: " + fieldName);
					}
				}
			}
			fieldMap.add("docs", terms.getDocCount());

		}
		if (fields != null
				&& (fields.contains(fieldName) || fields.contains("*"))) {
			getDetailedFieldInfo(req, fieldName, fieldMap);
		}
		// Add the field
		vInfo.add(fieldName, fieldMap);
		aInfo.add(AlfrescoSolrDataModel.getInstance()
				.getAlfrescoPropertyFromSchemaField(fieldName), fieldMap);
	}

	SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<>();
	finfo.addAll(vInfo);
	// finfo.add("mimetype()", finfo.get("cm:content.mimetype"));
	// finfo.add("contentSize()", finfo.get("cm:content.size"));
	finfo.addAll(aInfo);
	return finfo;
}
 

protected Query getFieldQuery(String field, String queryText, boolean quoted, boolean raw) throws SyntaxError {
  checkNullField(field);

  SchemaField sf;
  if (field.equals(lastFieldName)) {
    // only look up the SchemaField on a field change... this helps with memory allocation of dynamic fields
    // and large queries like foo_i:(1 2 3 4 5 6 7 8 9 10) when we are passed "foo_i" each time.
    sf = lastField;
  } else {
    // intercept magic field name of "_" to use as a hook for our
    // own functions.
    if (allowSubQueryParsing && field.charAt(0) == '_' && parser != null) {
      MagicFieldName magic = MagicFieldName.get(field);
      if (null != magic) {
        subQParser = parser.subQuery(queryText, magic.subParser);
        return subQParser.getQuery();
      }
    }

    lastFieldName = field;
    sf = lastField = schema.getFieldOrNull(field);
  }

  if (sf != null) {
    FieldType ft = sf.getType();
    // delegate to type for everything except tokenized fields
    if (ft.isTokenized() && sf.indexed()) {
      boolean fieldAutoGenPhraseQueries = ft instanceof TextField && ((TextField)ft).getAutoGeneratePhraseQueries();
      boolean fieldEnableGraphQueries = ft instanceof TextField && ((TextField)ft).getEnableGraphQueries();
      SynonymQueryStyle synonymQueryStyle = AS_SAME_TERM;
      if (ft instanceof TextField) {
        synonymQueryStyle = ((TextField)(ft)).getSynonymQueryStyle();
      }
      return newFieldQuery(getAnalyzer(), field, queryText, quoted, fieldAutoGenPhraseQueries, fieldEnableGraphQueries, synonymQueryStyle);
    } else {
      if (raw) {
        return new RawQuery(sf, queryText);
      } else {
        return ft.getFieldQuery(parser, sf, queryText);
      }
    }
  }

  // default to a normal field query
  return newFieldQuery(getAnalyzer(), field, queryText, quoted, false, true, AS_SAME_TERM);
}
 

/**
 * This method will force the appropriate facet method even if the user provided a different one as a request parameter
 *
 * N.B. this method could overwrite what you passed as request parameter. Be Extra careful
 *
 * @param field field we are faceting
 * @param method the facet method passed as a request parameter
 * @param mincount the minimum value a facet should have to be returned
 * @return the FacetMethod to use
 */
 static FacetMethod selectFacetMethod(SchemaField field, FacetMethod method, Integer mincount) {

   FieldType type = field.getType();
   if (type.isPointField()) {
     // Only FCS is supported for PointFields for now
     return FacetMethod.FCS;
   }

   /*The user did not specify any preference*/
   if (method == null) {
     /* Always use filters for booleans if not DocValues only... we know the number of values is very small. */
     if (type instanceof BoolField && (field.indexed() == true || field.hasDocValues() == false)) {
       method = FacetMethod.ENUM;
     } else if (type.getNumberType() != null && !field.multiValued()) {
      /* the per-segment approach is optimal for numeric field types since there
         are no global ords to merge and no need to create an expensive
         top-level reader */
       method = FacetMethod.FCS;
     } else {
       // TODO: default to per-segment or not?
       method = FacetMethod.FC;
     }
   }

   /* FC without docValues does not support single valued numeric facets */
   if (method == FacetMethod.FC
       && type.getNumberType() != null && !field.multiValued()) {
     method = FacetMethod.FCS;
   }

   /* UIF without DocValues can't deal with mincount=0, the reason is because
       we create the buckets based on the values present in the result set.
       So we are not going to see facet values which are not in the result set */
   if (method == FacetMethod.UIF
       && !field.hasDocValues() && mincount == 0) {
     method = field.multiValued() ? FacetMethod.FC : FacetMethod.FCS;
   }

   /* Unless isUninvertible() is true, we prohibit any use of UIF...
      Here we just force FC(S) instead, and trust that the DocValues faceting logic will
      do the right thing either way (with or w/o docvalues) */
   if (FacetMethod.UIF == method && ! field.isUninvertible()) {
     method = field.multiValued() ? FacetMethod.FC : FacetMethod.FCS;
   }
   
   /* ENUM can't deal with trie fields that index several terms per value */
   if (method == FacetMethod.ENUM
       && TrieField.getMainValuePrefix(type) != null) {
     method = field.multiValued() ? FacetMethod.FC : FacetMethod.FCS;
   }

   /* FCS can't deal with multi token fields */
   final boolean multiToken = field.multiValued() || type.multiValuedFieldCache();
   if (method == FacetMethod.FCS
       && multiToken) {
     method = FacetMethod.FC;
   }

   return method;
}
 

@Override
@SuppressWarnings("rawtypes")
public FacetProcessor createFacetProcessor(FacetContext fcontext) {
  SchemaField sf = fcontext.searcher.getSchema().getField(field);
  FieldType ft = sf.getType();
  boolean multiToken = sf.multiValued() || ft.multiValuedFieldCache();

  if (fcontext.facetInfo != null) {
    // refinement... we will end up either skipping the entire facet, or doing calculating only specific facet buckets
    if (multiToken && !sf.hasDocValues() && method!=FacetMethod.DV && sf.isUninvertible()) {
      // Match the access method from the first phase.
      // It won't always matter, but does currently for an all-values bucket
      return new FacetFieldProcessorByArrayUIF(fcontext, this, sf);
    }
    return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
  }

  NumberType ntype = ft.getNumberType();
  // ensure we can support the requested options for numeric faceting:
  if (ntype != null) {
    if (prefix != null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "Doesn't make sense to set facet prefix on a numeric field");
    }
    if (mincount == 0) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "Numeric fields do not support facet mincount=0; try indexing as terms");
      // TODO if indexed=true then we could add support
    }
  }

  // TODO auto-pick ENUM/STREAM SOLR-9351 when index asc and DocSet cardinality is *not* much smaller than term cardinality
  if (method == FacetMethod.ENUM) {// at the moment these two are the same
    method = FacetMethod.STREAM;
  }
  if (method == FacetMethod.STREAM && sf.indexed() && !ft.isPointField() &&
      // wether we can use stream processing depends on wether this is a shard request, wether
      // re-sorting has been requested, and if the effective sort during collection is "index asc"
      ( fcontext.isShard()
        // for a shard request, the effective per-shard sort must be index asc
        ? FacetSort.INDEX_ASC.equals(null == prelim_sort ? sort : prelim_sort)
        // for a non-shard request, we can only use streaming if there is no pre-sorting
        : (null == prelim_sort && FacetSort.INDEX_ASC.equals( sort ) ) ) ) {
        
    return new FacetFieldProcessorByEnumTermsStream(fcontext, this, sf);
  }

  // TODO if method=UIF and not single-valued numerics then simply choose that now? TODO add FieldType.getDocValuesType()

  if (!multiToken) {
    if (mincount > 0 && prefix == null && (ntype != null || method == FacetMethod.DVHASH)) {
      // TODO can we auto-pick for strings when term cardinality is much greater than DocSet cardinality?
      //   or if we don't know cardinality but DocSet size is very small
      return new FacetFieldProcessorByHashDV(fcontext, this, sf);
    } else if (ntype == null) {
      // single valued string...
      return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
    } else {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
          "Couldn't pick facet algorithm for field " + sf);
    }
  }

  if (sf.hasDocValues() && sf.getType().isPointField()) {
    return new FacetFieldProcessorByHashDV(fcontext, this, sf);
  }

  // multi-valued after this point

  if (sf.hasDocValues() || method == FacetMethod.DV || !sf.isUninvertible()) {
    // single and multi-valued string docValues
    return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
  }

  // Top-level multi-valued field cache (UIF)
  return new FacetFieldProcessorByArrayUIF(fcontext, this, sf);
}
 

private static SimpleOrderedMap<Object> getIndexedFieldsInfo(SolrQueryRequest req)
    throws Exception {

  SolrIndexSearcher searcher = req.getSearcher();
  SolrParams params = req.getParams();

  Set<String> fields = null;
  String fl = params.get(CommonParams.FL);
  if (fl != null) {
    fields = new TreeSet<>(Arrays.asList(fl.split( "[,\\s]+" )));
  }

  LeafReader reader = searcher.getSlowAtomicReader();
  IndexSchema schema = searcher.getSchema();

  // Don't be tempted to put this in the loop below, the whole point here is to alphabetize the fields!
  Set<String> fieldNames = new TreeSet<>();
  for(FieldInfo fieldInfo : reader.getFieldInfos()) {
    fieldNames.add(fieldInfo.name);
  }

  // Walk the term enum and keep a priority queue for each map in our set
  SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<>();

  for (String fieldName : fieldNames) {
    if (fields != null && ! fields.contains(fieldName) && ! fields.contains("*")) {
      continue; //we're not interested in this field Still an issue here
    }

    SimpleOrderedMap<Object> fieldMap = new SimpleOrderedMap<>();

    SchemaField sfield = schema.getFieldOrNull( fieldName );
    FieldType ftype = (sfield==null)?null:sfield.getType();

    fieldMap.add( "type", (ftype==null)?null:ftype.getTypeName() );
    fieldMap.add("schema", getFieldFlags(sfield));
    if (sfield != null && schema.isDynamicField(sfield.getName()) && schema.getDynamicPattern(sfield.getName()) != null) {
      fieldMap.add("dynamicBase", schema.getDynamicPattern(sfield.getName()));
    }
    Terms terms = reader.terms(fieldName);
    if (terms == null) { // Not indexed, so we need to report what we can (it made it through the fl param if specified)
      finfo.add( fieldName, fieldMap );
      continue;
    }

    if(sfield != null && sfield.indexed() ) {
      if (params.getBool(INCLUDE_INDEX_FIELD_FLAGS,true)) {
        Document doc = getFirstLiveDoc(terms, reader);

        if (doc != null) {
          // Found a document with this field
          try {
            IndexableField fld = doc.getField(fieldName);
            if (fld != null) {
              fieldMap.add("index", getFieldFlags(fld));
            } else {
              // it is a non-stored field...
              fieldMap.add("index", "(unstored field)");
            }
          } catch (Exception ex) {
            log.warn("error reading field: {}", fieldName);
          }
        }
      }
      fieldMap.add("docs", terms.getDocCount());
    }
    if (fields != null && (fields.contains(fieldName) || fields.contains("*"))) {
      getDetailedFieldInfo(req, fieldName, fieldMap);
    }
    // Add the field
    finfo.add( fieldName, fieldMap );
  }
  return finfo;
}
 

@Test
public void testClientErrorOnMalformedDate() throws Exception {
  final String BAD_VALUE = "NOT_A_DATE";
  ignoreException(BAD_VALUE);

  final List<String> FIELDS = new LinkedList<>();
  for (String type : new String[] {
      "tdt", "tdt1", "tdtdv", "tdtdv1",
      "dt_dv", "dt_dvo", "dt", "dt1", "dt_os"
      }) {
    FIELDS.add("malformed_" + type);
  }

  // test that malformed numerics cause client error not server error
  for (String field : FIELDS) {
    SolrException e1 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a bad date: " + field,
        () -> h.update(add( doc("id","100", field, BAD_VALUE))));
    String msg1 = e1.getMessage();
    assertTrue("not an (update) client error on field: " + field +" : "+ msg1,
        400 <= e1.code() && e1.code() < 500);
    assertTrue("(update) client error does not mention bad value: " + msg1,
        msg1.contains(BAD_VALUE));
    assertTrue("client error does not mention document id: " + msg1,
        msg1.contains("[doc=100]"));
    SchemaField sf = h.getCore().getLatestSchema().getField(field);
    if (!sf.hasDocValues() && !sf.indexed()) {
      continue;
    }
    SolrException e2 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a bad date: " + field,
        () -> h.query(req("q",field + ":" + BAD_VALUE))
    );
    String msg2 = e2.toString();
    assertTrue("not a (search) client error on field: " + field +" : "+ msg2,
        400 <= e2.code() && e2.code() < 500);
    assertTrue("(search) client error does not mention bad value: " + msg2,
        msg2.contains(BAD_VALUE));

    SolrException e3 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a bad date: " + field,
        () -> h.query(req("q",field + ":[NOW TO " + BAD_VALUE + "]"))
    );
    String msg3 = e3.toString();
    assertTrue("not a (search) client error on field: " + field +" : "+ msg3,
        400 <= e3.code() && e3.code() < 500);
    assertTrue("(search) client error does not mention bad value: " + msg3,
        msg3.contains(BAD_VALUE));
  }
}
 

@Test
public void testClientErrorOnMalformedNumbers() throws Exception {

  final String BAD_VALUE = "NOT_A_NUMBER";
  ignoreException(BAD_VALUE);

  final List<String> FIELDS = new LinkedList<>();
  for (String type : new String[] {
      "ti", "tf", "td", "tl",
      "i", "f", "d", "l",
      "i_dv", "f_dv", "d_dv", "l_dv",
      "i_dvo", "f_dvo", "d_dvo", "l_dvo",
      "i_os", "f_os", "d_os", "l_os"
      }) {
    FIELDS.add("malformed_" + type);
  }

  // test that malformed numerics cause client error not server error
  for (String field : FIELDS) {
    SolrException e1 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a non-number: " + field,
        () -> h.update(add( doc("id","100", field, BAD_VALUE))));
    String msg1 = e1.toString();
    assertTrue("not an (update) client error on field: " + field +" : "+ msg1,
        400 <= e1.code() && e1.code() < 500);
    assertTrue("(update) client error does not mention bad value: " + msg1,
        msg1.contains(BAD_VALUE));
    assertTrue("client error does not mention document id",
        msg1.contains("[doc=100]"));
    SchemaField sf = h.getCore().getLatestSchema().getField(field); 
    if (!sf.hasDocValues() && !sf.indexed()) {
      continue;
    }

    SolrException e2 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a non-number: " + field,
        () -> h.query(req("q",field + ":" + BAD_VALUE))
    );
    String msg2 = e2.toString();
    assertTrue("not a (search) client error on field: " + field +" : "+ msg2,
        400 <= e2.code() && e2.code() < 500);
    assertTrue("(search) client error does not mention bad value: " + msg2,
        msg2.contains(BAD_VALUE));

    SolrException e3 = expectThrows(SolrException.class,
        "Didn't encounter an error trying to add a non-number: " + field,
        () -> h.query(req("q",field + ":[10 TO " + BAD_VALUE + "]"))
    );
    String msg3 = e3.toString();
    assertTrue("not a (search) client error on field: " + field +" : "+ msg3,
        400 <= e3.code() && e3.code() < 500);
    assertTrue("(search) client error does not mention bad value: " + msg3,
        msg3.contains(BAD_VALUE));
  }
}
 

/**
 * Given a schema field, return whether or not such a field is supported for an in-place update.
 * Note: If an update command has updates to only supported fields (and _version_ is also supported),
 * only then is such an update command executed as an in-place update.
 */
public static boolean isSupportedFieldForInPlaceUpdate(SchemaField schemaField) {
  return !(schemaField.indexed() || schemaField.stored() || !schemaField.hasDocValues() || 
      schemaField.multiValued() || !(schemaField.getType() instanceof NumericValueFieldType));
}