org.apache.solr.schema.IndexSchema Java Examples

public void testParseFloatNonRootLocale() throws Exception {
  final DecimalFormatSymbols fr_FR = DecimalFormatSymbols.getInstance(new Locale("fr","FR"));
  final char groupChar = fr_FR.getGroupingSeparator();
  final char decimalChar = fr_FR.getDecimalSeparator();

  float value = 10898.83491F;
  String floatString1 = "10898"+decimalChar+"83491";
  String floatString2 = "10"+groupChar+"898"+decimalChar+"83491";
  
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull("float_f")); // should match dynamic field "*_f"
  assertNull(schema.getFieldOrNull("not_in_schema"));
  SolrInputDocument d = processAdd("parse-float-french-no-run-processor",
      doc(f("id", "140"), f("float_f", floatString1),
          f("not_in_schema", floatString2)));
  assertNotNull(d);
  assertThat(d.getFieldValue("float_f"), IS_FLOAT);
  assertEquals(value, (Float)d.getFieldValue("float_f"), EPSILON);
  assertThat(d.getFieldValue("not_in_schema"), IS_FLOAT);
  assertEquals(value, (Float)d.getFieldValue("not_in_schema"), EPSILON);
}
 

public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
  HashKey[] hashKeys = new HashKey[keys.length];
  SolrIndexSearcher searcher = (SolrIndexSearcher)indexSearcher;
  IndexSchema schema = searcher.getSchema();
  for(int i=0; i<keys.length; i++) {
    String key = keys[i];
    FieldType ft = schema.getField(key).getType();
    HashKey h = null;
    if(ft instanceof StrField) {
      h = new BytesHash(key, ft);
    } else {
      h = new NumericHash(key);
    }
    hashKeys[i] = h;
  }
  HashKey k = (hashKeys.length > 1) ? new CompositeHash(hashKeys) : hashKeys[0];
  return new HashCollector(k, workers, worker);
}
 

/**
 * Generates an NamedList of Explanations for each item in a list of docs.
 *
 * @param query The Query you want explanations in the context of
 * @param docs The Documents you want explained relative that query
 */
public static NamedList<Explanation> getExplanations
  (Query query,
   DocList docs,
   SolrIndexSearcher searcher,
   IndexSchema schema) throws IOException {

  NamedList<Explanation> explainList = new SimpleOrderedMap<>();
  DocIterator iterator = docs.iterator();
  for (int i=0; i<docs.size(); i++) {
    int id = iterator.nextDoc();

    Document doc = searcher.doc(id);
    String strid = schema.printableUniqueKey(doc);

    explainList.add(strid, searcher.explain(query, id) );
  }
  return explainList;
}
 

private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) {
  if (null != coreDescriptor.getCloudDescriptor()) {
    // we are evidently running in cloud mode.  
    //
    // In cloud mode, version field is required for correct consistency
    // ideally this check would be more fine grained, and individual features
    // would assert it when they initialize, but DistributedUpdateProcessor
    // is currently a big ball of wax that does more then just distributing
    // updates (ie: partial document updates), so it needs to work in no cloud
    // mode as well, and can't assert version field support on init.

    try {
      VersionInfo.getAndCheckVersionField(schema);
    } catch (SolrException e) {
      throw new SolrException(ErrorCode.SERVER_ERROR,
          "Schema will not work with SolrCloud mode: " +
              e.getMessage(), e);
    }
  }
}
 

public PeerSyncTest() {
  stress = 0;

  // TODO: a better way to do this?
  configString = "solrconfig-tlog.xml";
  schemaString = "schema.xml";

  // validate that the schema was not changed to an unexpected state
  try {
    initCore(configString, schemaString);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  IndexSchema schema = h.getCore().getLatestSchema();
  assertTrue(schema.getFieldOrNull("_version_").hasDocValues() && !schema.getFieldOrNull("_version_").indexed()
      && !schema.getFieldOrNull("_version_").stored());
  assertTrue(!schema.getFieldOrNull("val_i_dvo").indexed() && !schema.getFieldOrNull("val_i_dvo").stored() &&
      schema.getFieldOrNull("val_i_dvo").hasDocValues());
}
 

private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      try {
        SchemaField field = schema.getField(fieldName);
        if (field.stored() && field.getType() instanceof StrField ) {
          strFields.add( fieldName );
        }
      }
      catch (Throwable e )
      {
          
      }
    }
  }
    
  return strFields;
}
 

private static SolrInputDocument toSolrInputDocument(Document doc, IndexSchema schema) {
  SolrInputDocument out = new SolrInputDocument();
  for( IndexableField f : doc.getFields() ) {
    String fname = f.name();
    SchemaField sf = schema.getFieldOrNull(f.name());
    Object val = null;
    if (sf != null) {
      if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue;
      val = sf.getType().toObject(f);   // object or external string?
    } else {
      val = f.stringValue();
      if (val == null) val = f.numericValue();
      if (val == null) val = f.binaryValue();
      if (val == null) val = f;
    }

    // todo: how to handle targets of copy fields (including polyfield sub-fields)?
    out.addField(fname, val);
  }
  return out;
}
 

/**
 * This is a destructive call... the queue is empty at the end
 */
public NamedList<Integer> toNamedList( IndexSchema schema )
{
  // reverse the list..
  List<TermInfo> aslist = new LinkedList<>();
  while( size() > 0 ) {
    aslist.add( 0, (TermInfo)pop() );
  }

  NamedList<Integer> list = new NamedList<>();
  for (TermInfo i : aslist) {
    String txt = i.term.text();
    SchemaField ft = schema.getFieldOrNull( i.term.field() );
    if( ft != null ) {
      txt = ft.getType().indexedToReadable( txt );
    }
    list.add( txt, i.docFreq );
  }
  return list;
}
 

public void testMixedFloats() throws Exception {
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull("float_tf")); // should match dynamic field "*_tf"
  Map<Float,Object> mixedFloats = new HashMap<>();
  mixedFloats.put(85.0f, "85");
  mixedFloats.put(2894518.0f, "2,894,518");
  mixedFloats.put(2.94423E-9f, 2.94423E-9f); // Float-typed field value
  mixedFloats.put(48794721.937f, "48,794,721.937");
  SolrInputDocument d = processAdd("parse-float-no-run-processor", 
                                   doc(f("id", "342"), f("float_tf", mixedFloats.values())));
  assertNotNull(d);
  for (Object o : d.getFieldValues("float_tf")) {
    assertThat(o, IS_FLOAT);
    mixedFloats.remove(o);
  }
  assertTrue(mixedFloats.isEmpty());
}
 

/**
 * Gets the field type.
 *
 * @param schema the schema
 * @param field the field
 * @return the field type
 * @throws IOException Signals that an I/O exception has occurred.
 */
private String getFieldType(IndexSchema schema, String field)
    throws IOException {
  SchemaField sf = schema.getField(field);
  FieldType ft = sf.getType();
  if (ft != null) {
    if (ft.isPointField() && !sf.hasDocValues()) {
      return ComponentFacet.TYPE_POINTFIELD_WITHOUT_DOCVALUES;
    }
    NumberType nt = ft.getNumberType();
    if (nt != null) {
      return nt.name();
    } else {
      return ComponentFacet.TYPE_STRING;
    }
  } else {
    // best guess
    return ComponentFacet.TYPE_STRING;
  }
}
 

public void testCanCreateTombstonesRequiredFieldInTombstoneConfig() {
  DocBasedVersionConstraintsProcessorFactory factory = new DocBasedVersionConstraintsProcessorFactory();
  NamedList<Object> config = new NamedList<>();
  config.add("versionField", "_version_");
  NamedList<Object> tombstoneConfig = new NamedList<>();
  config.add("tombstoneConfig", tombstoneConfig);
  tombstoneConfig.add("sku1", "foo");
  factory.init(config);
  IndexSchema schema = h.getCore().getLatestSchema();
  SchemaField sf = schema.getField("sku1");
  assertThat(sf, is(not(nullValue())));
  assertThat(schema.getRequiredFields(), not(hasItem(sf)));
  try {
    schema.getRequiredFields().add(sf);
    assertThat(factory.canCreateTombstoneDocument(schema), is(true));
  } finally {
    schema.getRequiredFields().remove(sf);
  }
}
 

private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      SchemaField field = schema.getField(fieldName);
      if (field.stored() && field.getType() instanceof StrField ) {
        strFields.add( fieldName );
      }
    }
  }
    
  return strFields;
}
 

public void testStringWithCopyFieldAndMaxChars() throws Exception {
  IndexSchema schema = h.getCore().getLatestSchema();
  final String fieldName = "stringField";
  final String strFieldName = fieldName+"_str";
  assertNull(schema.getFieldOrNull(fieldName));
  String content = "This is a text that should be copied to a string field and cutoff at 10 characters";
  SolrInputDocument d = processAdd("add-fields-maxchars", doc(f("id", "1"), f(fieldName, content)));
  assertNotNull(d);
  System.out.println("Document is "+d);
  schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull(fieldName));
  assertNotNull(schema.getFieldOrNull(strFieldName));
  assertEquals("text", schema.getFieldType(fieldName).getTypeName());
  // We have three copyFields, one with maxChars 10 and two with maxChars 20
  assertEquals(3, schema.getCopyFieldProperties(true, Collections.singleton(fieldName), null).size());
  assertEquals("The configured maxChars cutoff does not exist on the copyField", 10, 
      schema.getCopyFieldProperties(true, Collections.singleton(fieldName), Collections.singleton(strFieldName))
          .get(0).get("maxChars"));
  assertEquals("The configured maxChars cutoff does not exist on the copyField", 20, 
      schema.getCopyFieldProperties(true, Collections.singleton(fieldName), Collections.singleton(fieldName+"_t"))
          .get(0).get("maxChars"));
  assertEquals("The configured maxChars cutoff does not exist on the copyField", 20, 
      schema.getCopyFieldProperties(true, Collections.singleton(fieldName), Collections.singleton(fieldName+"2_t"))
          .get(0).get("maxChars"));
}
 

public void testNextCursorMark() throws IOException {
  final Collection<String> allFieldNames = getAllFieldNames();
  final SolrQueryRequest req = req();
  final IndexSchema schema = req.getSchema();

  final String randomSortString = CursorPagingTest.buildRandomSort(allFieldNames);
  final SortSpec ss = SortSpecParsing.parseSortSpec(randomSortString, req);

  final CursorMark previous = new CursorMark(schema, ss);
  previous.parseSerializedTotem(CURSOR_MARK_START);

  List<Object> nextValues = Arrays.<Object>asList(buildRandomSortObjects(ss));
  final CursorMark next = previous.createNext(nextValues);
  assertEquals("next values not correct", nextValues, next.getSortValues());
  assertEquals("next SortSpec not correct", ss, next.getSortSpec());

  try {
    // append to our random sort string so we know it has wrong num clauses
    final SortSpec otherSort = SortSpecParsing.parseSortSpec(randomSortString+",id asc", req);
    CursorMark trash = previous.createNext(Arrays.<Object>asList
                                           (buildRandomSortObjects(otherSort)));
    fail("didn't fail on next with incorrect num of sortvalues");
  } catch (AssertionError e) {
    // NOOP: we're happy
  }
}
 

public void testFailedParseMixedLong() throws Exception {
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNull(schema.getFieldOrNull("not_in_schema"));
  Map<Object,Object> mixed = new HashMap<>();
  Float floatVal = 294423.0f;
  mixed.put(85L, "85");
  mixed.put(floatVal, floatVal); // Float-typed field value
  mixed.put(-2894518L, "-2,894,518");
  mixed.put(1879472193L, "1,879,472,193");
  SolrInputDocument d = processAdd("parse-long-no-run-processor",
                                   doc(f("id", "7204"), f("not_in_schema", mixed.values())));
  assertNotNull(d);
  boolean foundFloat = false;
  for (Object o : d.getFieldValues("not_in_schema")) {
    if (floatVal == o) {
      foundFloat = true;
    } else {
      assertThat(o, IS_STRING);
    }
    mixed.values().remove(o);
  }
  assertTrue(foundFloat);
  assertTrue(mixed.isEmpty());
}
 

private void iterateAddDomainTermFields(boolean isBoosted, String domainTermsFieldName,
                                        IndexSchema indexSchema, Document doc,
                                        List<Pair<String, Double>> filteredCandidateTerms) {
    // remove previous fields if exists
    doc.removeFields(domainTermsFieldName);

    for (Pair<String, Double> filteredTerm : filteredCandidateTerms) {
        if (filteredTerm == null) {
            continue;
        }

        if (isBoosted) {
            doc.add(indexSchema.getField(domainTermsFieldName).createField(filteredTerm.first()
                    ));
        } else {
            doc.add(indexSchema.getField(domainTermsFieldName).createField(filteredTerm.first()
                    ));
        }
    }
}
 

IndexSchema addFields(IndexSchema schema) {

    List<SchemaField> fieldsToAdd = new ArrayList<>();

    for (RetrieveField field : fields.values()) {
      allFields.add(field);
      SchemaField schemaField = field.schemaField;
      fieldsToAdd.add(schemaField);
      if (schemaField.multiValued()) {
        multiValuedFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored() == false) {
        dvNotStoredFields.add(field);
      }
      if (schemaField.hasDocValues() == false && schemaField.stored()) {
        storedNotDvFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored()) {
        storedAndDvFields.add(field);
      }
      if (schemaField.stored() && schemaField.multiValued()) {
        storedMvFields.add(field);
      }
    }
    return schema.addFields(fieldsToAdd, Collections.emptyMap(), false);
  }
 

public void testParseBooleanRoundTrip() throws Exception {
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull("boolean1_b")); // should match dynamic field "*_b"
  assertNotNull(schema.getFieldOrNull("boolean2_b")); // should match dynamic field "*_b"
  boolean value1 = true;
  boolean value2 = false;
  SolrInputDocument d = processAdd("parse-boolean",
      doc(f("id", "141"), f("boolean1_b", value1), f("boolean2_b", value2)));
  assertNotNull(d);
  assertThat(d.getFieldValue("boolean1_b"), IS_BOOLEAN);
  assertEquals(value1, d.getFieldValue("boolean1_b"));
  assertThat(d.getFieldValue("boolean2_b"), IS_BOOLEAN);
  assertEquals(value2, d.getFieldValue("boolean2_b"));

  assertU(commit());
  assertQ(req("id:141")
      ,"//bool[@name='boolean1_b'][.='" + value1 + "']"
      ,"//bool[@name='boolean2_b'][.='" + value2 + "']");
}
 

public void testParseTrieDoubleRoundTrip() throws Exception {
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull("double1_td")); // should match dynamic field "*_td"
  assertNotNull(schema.getFieldOrNull("double2_td")); // should match dynamic field "*_td"
  double value = 10898.83491;
  String doubleString1 = "10898.83491";
  String doubleString2 = "10,898.83491";
  SolrInputDocument d = processAdd("parse-double",
      doc(f("id", "728"), f("double1_td", doubleString1), f("double2_td", doubleString2)));
  assertNotNull(d);
  assertThat(d.getFieldValue("double1_td"), IS_DOUBLE);
  assertEquals(value, (Double)d.getFieldValue("double1_td"), EPSILON);
  assertThat(d.getFieldValue("double2_td"), IS_DOUBLE);
  assertEquals(value, (Double)d.getFieldValue("double2_td"), EPSILON);

  assertU(commit());
  assertQ(req("id:728")
      ,"//double[@name='double1_td'][.='" + value + "']"
      ,"//double[@name='double2_td'][.='" + value + "']");
}
 

public void testParseLongNonRootLocale() throws Exception {
  final DecimalFormatSymbols ru_RU = DecimalFormatSymbols.getInstance(new Locale("ru","RU"));
  final char groupChar = ru_RU.getGroupingSeparator();
  
  long value = 1089883491L;
  String longString1 = "1089883491";
  String longString2 = "1"+groupChar+"089"+groupChar+"883"+groupChar+"491";
  
  IndexSchema schema = h.getCore().getLatestSchema();
  assertNotNull(schema.getFieldOrNull("long_l")); // should match dynamic field "*_l"
  assertNull(schema.getFieldOrNull("not_in_schema"));
  SolrInputDocument d = processAdd("parse-long-russian-no-run-processor",
                                   doc(f("id", "113"), f("long_l", longString1), f("not_in_schema", longString2)));
  assertNotNull(d);
  assertThat(d.getFieldValue("long_l"), IS_LONG);
  assertEquals(value, ((Long)d.getFieldValue("long_l")).longValue());
  assertThat(d.getFieldValue("not_in_schema"), IS_LONG);
  assertEquals(value, ((Long)d.getFieldValue("not_in_schema")).longValue());
}
 

/**
 * Utility method that can be used to define a FieldNameSelector
 * using the same types of rules as the FieldMutatingUpdateProcessor init 
 * code.  This may be useful for Factories that wish to define default 
 * selectors in similar terms to what the configuration would look like.
 * Uses {@code schema} for checking field existence.
 * @lucene.internal
 */
public static FieldNameSelector createFieldNameSelector
  (final SolrResourceLoader loader,
   final IndexSchema schema,
   final SelectorParams params,
   final FieldNameSelector defSelector) {

  if (params.noSelectorsSpecified()) {
    return defSelector;
  }

  final ConfigurableFieldNameSelectorHelper helper =
    new ConfigurableFieldNameSelectorHelper(loader, params);
  return fieldName -> helper.shouldMutateBasedOnSchema(fieldName, schema);
}
 

@Test
public void testManagedIndexSchemaCreation() {
  //Solr locator should select EmbeddedSolrServer only solrHome is specified
  SolrLocator solrLocator = new SolrLocator(new SolrMorphlineContext.Builder().build());
  solrLocator.setSolrHomeDir(RESOURCES_DIR + "/solr/managedSchemaCollection");
  solrLocator.setCollectionName("example-managed");

  IndexSchema indexSchema = solrLocator.getIndexSchema();

  assertNotNull(indexSchema);
  assertTrue(indexSchema instanceof ManagedIndexSchema);
  
  assertEquals("example-managed", indexSchema.getSchemaName());
}
 

@SuppressWarnings({"unchecked"})
public static void doStandardResultsDebug(
        SolrQueryRequest req,
        Query query,
        DocList results,
        boolean dbgResults,
        @SuppressWarnings({"rawtypes"})NamedList dbg) throws IOException
{
  if (dbgResults) {
    SolrIndexSearcher searcher = req.getSearcher();
    IndexSchema schema = searcher.getSchema();
    boolean explainStruct = req.getParams().getBool(CommonParams.EXPLAIN_STRUCT, false);

    if (results != null) {
      NamedList<Explanation> explain = getExplanations(query, results, searcher, schema);
      dbg.add("explain", explainStruct
          ? explanationsToNamedLists(explain)
          : explanationsToStrings(explain));
    }

    String otherQueryS = req.getParams().get(CommonParams.EXPLAIN_OTHER);
    if (otherQueryS != null && otherQueryS.length() > 0) {
      DocList otherResults = doSimpleQuery(otherQueryS, req, 0, 10);
      dbg.add("otherQuery", otherQueryS);
      NamedList<Explanation> explainO = getExplanations(query, otherResults, searcher, schema);
      dbg.add("explainOther", explainStruct
              ? explanationsToNamedLists(explainO)
              : explanationsToStrings(explainO));
    }
  }
}
 

public SolrIndexSearcher(SolrCore core, String path, IndexSchema schema, SolrIndexConfig config, String name,
    boolean enableCache, DirectoryFactory directoryFactory) throws IOException {
  // We don't need to reserve the directory because we get it from the factory
  this(core, path, schema, name, getReader(core, config, directoryFactory, path), true, enableCache, false,
      directoryFactory);
  // Release the directory at close.
  this.releaseDirectory = true;
}
 

public String getId() {
  if (id == null && indexedId != null) {
    IndexSchema schema = req.getSchema();
    SchemaField sf = schema.getUniqueKeyField();
    if (sf != null) {
      CharsRefBuilder ref = new CharsRefBuilder();
      sf.getType().indexedToReadable(indexedId, ref);
      id = ref.toString();
    }
  }
  return id;
}
 

/**
 *
 * Merges an Atomic Update inside a document hierarchy
 * @param sdoc the doc containing update instructions
 * @param oldDocWithChildren the doc (children included) before the update
 * @param sdocWithChildren the updated doc prior to the update (children included)
 * @return root doc (children included) after update
 */
public SolrInputDocument mergeChildDoc(SolrInputDocument sdoc, SolrInputDocument oldDocWithChildren,
                                       SolrInputDocument sdocWithChildren) {
  // get path of document to be updated
  String updatedDocPath = (String) sdocWithChildren.getFieldValue(IndexSchema.NEST_PATH_FIELD_NAME);
  // get the SolrInputField containing the document which the AddUpdateCommand updates
  SolrInputField sifToReplace = getFieldFromHierarchy(oldDocWithChildren, updatedDocPath);
  // update SolrInputField, either appending or replacing the updated document
  updateDocInSif(sifToReplace, sdocWithChildren, sdoc);
  return oldDocWithChildren;
}
 

public UninvertDocValuesMergePolicyFactory(SolrResourceLoader resourceLoader, MergePolicyFactoryArgs args, IndexSchema schema) {
  super(resourceLoader, args, schema);
  final Boolean sic = (Boolean)args.remove("skipIntegrityCheck");
  if (sic != null) {
    this.skipIntegrityCheck = sic.booleanValue();
  } else {
    this.skipIntegrityCheck = false;
  }
  if (!args.keys().isEmpty()) {
    throw new IllegalArgumentException("Arguments were "+args+" but "+getClass().getSimpleName()+" takes no arguments.");
  }
}
 

private static String getID( SolrInputDocument doc, IndexSchema schema )
{
  String id = "";
  SchemaField sf = schema.getUniqueKeyField();
  if( sf != null ) {
    id = "[doc="+doc.getFieldValue( sf.getName() )+"] ";
  }
  return id;
}
 

protected void checkFieldsInSchema(SolrIndexSearcher searcher, Collection<String> fields) throws SolrException {
	IndexSchema schema = searcher.getSchema();
	for (String field : fields) {
		SchemaField sField = schema.getField(field);
		if (sField == null) {
			throw new SolrException(ErrorCode.BAD_REQUEST, "\"" + field
					+ "\" is not in schema " + schema.getSchemaName());
		}
	}
}
 

public MLAnalayser(MLAnalysisMode mlAnalaysisMode, IndexSchema schema, Mode mode)
{
    super(Analyzer.PER_FIELD_REUSE_STRATEGY);
    this.mlAnalaysisMode = mlAnalaysisMode;
    this.schema = schema;
    this.mode = mode;
}