Java Code Examples for org.apache.solr.core.SolrCore#getLatestSchema()

@Test
public void testMockExchangeRateProvider() throws Exception {
  assumeTrue("This test is only applicable to the mock exchange rate provider",
      expectedProviderClass.equals(MockExchangeRateProvider.class));
  
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  SchemaField field = schema.getField(fieldName);
  FieldType fieldType = field.getType();
  ExchangeRateProvider provider = ((CurrencyFieldType)fieldType).getProvider();

  // A few tests on the provider directly
  assertEquals(3, provider.listAvailableCurrencies().size());
  assertTrue(provider.reload());
  assertEquals(0.8, provider.getExchangeRate("USD", "EUR"), 0.00000000001);
}
 

public void populateValueClasses(SolrCore core) {
  IndexSchema schema = core.getLatestSchema();
  ClassLoader loader = core.getResourceLoader().getClassLoader();
  if (null == schema.getFieldTypeByName(fieldTypeName)) {
    throw new SolrException(SERVER_ERROR, "fieldType '" + fieldTypeName + "' not found in the schema");
  }
  valueClasses = new HashSet<>();
  for (String valueClassName : valueClassNames) {
    try {
      valueClasses.add(loader.loadClass(valueClassName));
    } catch (ClassNotFoundException e) {
      throw new SolrException(SERVER_ERROR,
          "valueClass '" + valueClassName + "' not found for fieldType '" + fieldTypeName + "'");
    }
  }
}
 

@Test
public void testSchemaLoadingComplexAnalyzer() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  assertTrue( schema.getFieldTypes().containsKey("text") );

  @SuppressWarnings({"unchecked"})
  SimpleOrderedMap<Object> indexAnalyzerProps =
      (SimpleOrderedMap<Object>)schema.getFieldTypeByName("text")
          .getNamedPropertyValues(true).get("indexAnalyzer");
  checkTokenizerName(indexAnalyzerProps, "whitespace");
  checkTokenFilterNames(indexAnalyzerProps, new String[]{"stop", "wordDelimiterGraph", "lowercase", "keywordMarker", "porterStem", "removeDuplicates", "flattenGraph"});

  @SuppressWarnings({"unchecked"})
  SimpleOrderedMap<Object> queryAnalyzerProps =
      (SimpleOrderedMap<Object>)schema.getFieldTypeByName("text")
          .getNamedPropertyValues(true).get("queryAnalyzer");
  checkTokenizerName(queryAnalyzerProps, "whitespace");
  checkTokenFilterNames(queryAnalyzerProps, new String[]{"synonymGraph", "stop", "wordDelimiterGraph", "lowercase", "keywordMarker", "porterStem", "removeDuplicates"});

  assertNotNull(schema.getFieldTypeByName("text").getIndexAnalyzer());
  assertNotNull(schema.getFieldTypeByName("text").getQueryAnalyzer());
}
 

@Test
public void testSearchDetails() throws Exception {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  double[] xy = new double[]{35.0, -79.34};
  String point = xy[0] + "," + xy[1];
  //How about some queries?
  //don't need a parser for this path currently.  This may change
  assertU(adoc("id", "0", "home_ns", point));
  assertU(commit());
  SchemaField home = schema.getField("home_ns");
  PointType pt = (PointType) home.getType();
  assertEquals(pt.getDimension(), 2);
  Query q = pt.getFieldQuery(null, home, point);
  assertNotNull(q);
  assertTrue(q instanceof BooleanQuery);
  //should have two clauses, one for 35.0 and the other for -79.34
  BooleanQuery bq = (BooleanQuery) q;
  assertEquals(2, bq.clauses().size());
  clearIndex();
}
 

@Test
public void testSchemaLoading() 
{
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  SchemaField uniqueKey = schema.getUniqueKeyField();
  
  assertFalse( uniqueKey.isRequired() );
  
  assertFalse( schema.getRequiredFields().contains( uniqueKey ) );
}
 

@SuppressForbidden(reason = "Need currentTimeMillis to get start time for request (to be used for stats/debugging)")
public SolrQueryRequestBase(SolrCore core, SolrParams params, RTimerTree requestTimer) {
  this.core = core;
  this.schema = null == core ? null : core.getLatestSchema();
  this.params = this.origParams = params;
  this.requestTimer = requestTimer;
  this.startTime = System.currentTimeMillis();
}
 

@Test
public void testSchemaLoadingAnalyzerWithCharFilters() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  assertTrue( schema.getFieldTypes().containsKey("charfilthtmlmap") );
  @SuppressWarnings({"unchecked"})
  SimpleOrderedMap<Object> analyzerProps =
      (SimpleOrderedMap<Object>)schema.getFieldTypeByName("charfilthtmlmap")
          .getNamedPropertyValues(true).get("analyzer");
  checkTokenizerName(analyzerProps, "whitespace");
  checkCharFilterNames(analyzerProps, new String[]{"htmlStrip", "mapping"});

  assertNotNull(schema.getFieldTypeByName("charfilthtmlmap").getIndexAnalyzer());
  assertNotNull(schema.getFieldTypeByName("charfilthtmlmap").getQueryAnalyzer());
}
 

@Test
public void testCurrencyFieldType() throws Exception {
  assumeTrue("This test is only applicable to the XML file based exchange rate provider",
      expectedProviderClass.equals(FileExchangeRateProvider.class));

  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  SchemaField amount = schema.getField(fieldName);
  assertNotNull(amount);
  assertTrue(fieldName + " is not a poly field", amount.isPolyField());
  FieldType tmp = amount.getType();
  assertTrue(fieldName + " is not an instance of CurrencyFieldType", tmp instanceof CurrencyFieldType);
  String currencyValue = "1.50,EUR";
  List<IndexableField> fields = amount.createFields(currencyValue);
  assertEquals(fields.size(), 3);

  // First field is currency code, second is value, third is stored.
  for (int i = 0; i < 3; i++) {
    boolean hasValue = fields.get(i).readerValue() != null
            || fields.get(i).numericValue() != null
            || fields.get(i).stringValue() != null;
    assertTrue("Doesn't have a value: " + fields.get(i), hasValue);
  }

  assertEquals(schema.getFieldTypeByName("string").toExternal(fields.get(2)), "1.50,EUR");
  
  // A few tests on the provider directly
  ExchangeRateProvider p = ((CurrencyFieldType)tmp).getProvider();
  Set<String> availableCurrencies = p.listAvailableCurrencies();
  assertEquals(5, availableCurrencies.size());
  assertTrue(p.reload());
  assertEquals(2.5, p.getExchangeRate("USD", "EUR"), 0.00000000001);
}
 

public void testExpectedProvider() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  SchemaField field = schema.getField(fieldName);
  FieldType fieldType = field.getType();
  ExchangeRateProvider provider = ((CurrencyFieldType)fieldType).getProvider();
  assertEquals(expectedProviderClass, provider.getClass());
}
 

/**
 * Returns true if the field doesn't match any schema field or dynamic field,
 *           or if the matched field's type is BoolField
 */
@Override
public FieldNameSelector getDefaultSelector(final SolrCore core) {
  return fieldName -> {
    final IndexSchema schema = core.getLatestSchema();
    FieldType type = schema.getFieldTypeNoEx(fieldName);
    return (null == type) || (type instanceof BoolField);
  };
}
 

@Override
public FieldNameSelector getDefaultSelector(final SolrCore core) {
  return fieldName -> {
    final IndexSchema schema = core.getLatestSchema();
    FieldType type = schema.getFieldTypeNoEx(fieldName);
    return (null == type);
  };
}
 

@Test
public void testRuntimeFieldCreation() {
  // any field manipulation needs to happen when you know the core will not
  // be accepting any requests.  Typically this is done within the inform()
  // method.  Since this is a single threaded test, we can change the fields
  // willi-nilly

  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  final String fieldName = "runtimefield";
  SchemaField sf = new SchemaField( fieldName, schema.getFieldTypes().get( "string" ) );
  schema.getFields().put( fieldName, sf );

  // also register a new copy field (from our new field)
  schema.registerCopyField( fieldName, "dynamic_runtime" );
  schema.refreshAnalyzers();

  assertU(adoc("id", "10", "title", "test", fieldName, "aaa"));
  assertU(commit());

  SolrQuery query = new SolrQuery( fieldName+":aaa" );
  query.set( "indent", "true" );
  SolrQueryRequest req = new LocalSolrQueryRequest( core, query );

  assertQ("Make sure they got in", req
          ,"//*[@numFound='1']"
          ,"//result/doc[1]/str[@name='id'][.='10']"
          );

  // Check to see if our copy field made it out safely
  query.setQuery( "dynamic_runtime:aaa" );
  assertQ("Make sure they got in", req
          ,"//*[@numFound='1']"
          ,"//result/doc[1]/str[@name='id'][.='10']"
          );
  clearIndex();
}
 

private ManagedIndexSchema getFreshManagedSchema(SolrCore core) throws IOException,
    KeeperException, InterruptedException {

  SolrResourceLoader resourceLoader = core.getResourceLoader();
  String name = core.getLatestSchema().getResourceName();
  if (resourceLoader instanceof ZkSolrResourceLoader) {
    final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader)resourceLoader;
    SolrZkClient zkClient = zkLoader.getZkController().getZkClient();
    try {
      if (!zkClient.exists(zkLoader.getConfigSetZkPath() + "/" + name, true)) {
        String backupName = name + ManagedIndexSchemaFactory.UPGRADED_SCHEMA_EXTENSION;
        if (!zkClient.exists(zkLoader.getConfigSetZkPath() + "/" + backupName, true)) {
          log.warn("Unable to retrieve fresh managed schema, neither {} nor {} exist.", name, backupName);
          // use current schema
          return (ManagedIndexSchema) core.getLatestSchema();
        } else {
          name = backupName;
        }
      }
    } catch (Exception e) {
      log.warn("Unable to retrieve fresh managed schema {}", name, e);
      // use current schema
      return (ManagedIndexSchema) core.getLatestSchema();
    }
    InputStream in = resourceLoader.openResource(name);
    if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
      int version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
      log.info("managed schema loaded . version : {} ", version);
      return new ManagedIndexSchema(core.getSolrConfig(), name, new InputSource(in), true, name, version,
          core.getLatestSchema().getSchemaUpdateLock());
    } else {
      return (ManagedIndexSchema) core.getLatestSchema();
    }
  } else {
    return (ManagedIndexSchema) core.getLatestSchema();
  }
}
 

@Test
public void testRequiredFieldsConfig() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  SchemaField uniqueKey = schema.getUniqueKeyField();

  // Make sure the uniqueKey is required
  assertTrue( uniqueKey.isRequired() ); 
  assertTrue( schema.getRequiredFields().contains( uniqueKey ) );
  
  // we specified one required field, but all devault valued fields are also required
  Collection<SchemaField> requiredFields =schema.getRequiredFields();
  int numDefaultFields = schema.getFieldsWithDefaultValue().size();
  assertEquals( numDefaultFields+1+1, requiredFields.size()); // also the uniqueKey
}
 

@Test
public void testSchemaLoadingSimpleAnalyzer() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  assertTrue( schema.getFieldTypes().containsKey("text_ws") );
  @SuppressWarnings({"unchecked"})
  SimpleOrderedMap<Object> analyzerProps =
      (SimpleOrderedMap<Object>)schema.getFieldTypeByName("text_ws")
      .getNamedPropertyValues(true).get("analyzer");
  checkTokenizerName(analyzerProps, "whitespace");

  assertNotNull(schema.getFieldTypeByName("text_ws").getIndexAnalyzer());
  assertNotNull(schema.getFieldTypeByName("text_ws").getQueryAnalyzer());
}
 

@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void inform(SolrCore core) {
  if (initParams != null) {
    log.info("Initializing spell checkers");
    boolean hasDefault = false;
    for (int i = 0; i < initParams.size(); i++) {
      if (initParams.getName(i).equals("spellchecker")) {
        Object cfg = initParams.getVal(i);
        if (cfg instanceof NamedList) {
          addSpellChecker(core, hasDefault, (NamedList) cfg);
        } else if (cfg instanceof Map) {
          addSpellChecker(core, hasDefault, new NamedList((Map) cfg));
        } else if (cfg instanceof List) {
          for (Object o : (List) cfg) {
            if (o instanceof Map) {
              addSpellChecker(core, hasDefault, new NamedList((Map) o));
            }
          }
        }
      }
    }

    Map<String, QueryConverter> queryConverters = new HashMap<>();
    core.initPlugins(queryConverters,QueryConverter.class);

    //ensure that there is at least one query converter defined
    if (queryConverters.size() == 0) {
      log.trace("No queryConverter defined, using default converter");
      queryConverters.put("queryConverter", new SpellingQueryConverter());
    }

    //there should only be one
    if (queryConverters.size() == 1) {
      queryConverter = queryConverters.values().iterator().next();
      IndexSchema schema = core.getLatestSchema();
      String fieldTypeName = (String) initParams.get("queryAnalyzerFieldType");
      FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
      Analyzer analyzer = fieldType == null ? new WhitespaceAnalyzer()
              : fieldType.getQueryAnalyzer();
      //TODO: There's got to be a better way!  Where's Spring when you need it?
      queryConverter.setAnalyzer(analyzer);
    }
  }
}
 

@Test
public void testSchemaLoading() {
  SolrCore core = h.getCore();
  IndexSchema schema = core.getLatestSchema();
  assertTrue( schema.getFieldTypes().containsKey("text_synonyms") );
}
 

private void loadExternalFileDictionary(SolrCore core, SolrIndexSearcher searcher) {
  try {
    IndexSchema schema = null == searcher ? core.getLatestSchema() : searcher.getSchema();
    // Get the field's analyzer
    if (fieldTypeName != null && schema.getFieldTypeNoEx(fieldTypeName) != null) {
      FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
      // Do index-time analysis using the given fieldType's analyzer
      Directory ramDir = new ByteBuffersDirectory();

      LogMergePolicy mp = new LogByteSizeMergePolicy();
      mp.setMergeFactor(300);

      IndexWriter writer = new IndexWriter(
          ramDir,
          new IndexWriterConfig(fieldType.getIndexAnalyzer()).
              setMaxBufferedDocs(150).
              setMergePolicy(mp).
              setOpenMode(IndexWriterConfig.OpenMode.CREATE)
              // TODO: if we enable this, codec gets angry since field won't exist in the schema
              // .setCodec(core.getCodec())
      );

      List<String> lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding);

      for (String s : lines) {
        Document d = new Document();
        d.add(new TextField(WORD_FIELD_NAME, s, Field.Store.NO));
        writer.addDocument(d);
      }
      writer.forceMerge(1);
      writer.close();

      dictionary = new HighFrequencyDictionary(DirectoryReader.open(ramDir),
              WORD_FIELD_NAME, 0.0f);
    } else {
      // check if character encoding is defined
      if (characterEncoding == null) {
        dictionary = new PlainTextDictionary(core.getResourceLoader().openResource(sourceLocation));
      } else {
        dictionary = new PlainTextDictionary(new InputStreamReader(core.getResourceLoader().openResource(sourceLocation), characterEncoding));
      }
    }


  } catch (IOException e) {
    log.error( "Unable to load spellings", e);
  }
}
 

@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
public void finishStage(ResponseBuilder rb) {
	System.out.println("===== FINISH STAGE =====");
	System.out.println("shards=" + rb.shards);

	SolrCore core = rb.req.getCore();
	IndexSchema schema = core.getLatestSchema();
	String uniqueKeyField = schema.getUniqueKeyField().getName();
	
	boolean includeShardId = fieldListIncludes(rb, DJOIN_FIELD);
	boolean includeShard = fieldListIncludes(rb, SHARD_FIELD);

	// only do this in final stage
	if (rb.stage != 3000) return;
	
	System.out.println("*** Federating results ***");
	
	SolrDocumentList feds = new SolrDocumentList();
	//rb.rsp.getValues().add("federated", feds);

	/*NamedList results = (NamedList)grouped.get(joinField);
	feds.setNumFound((Integer)results.get("matches"));
	feds.setStart(rb.getQueryCommand().getOffset());
	List<NamedList> groups = (List<NamedList>)results.get("groups");
	for (NamedList group : groups) {
		SolrDocumentList docs = (SolrDocumentList)group.get("doclist");
		SolrDocument superDoc = new SolrDocument();
		for (SolrDocument doc : docs) {
			for (String fieldName : doc.getFieldNames()) {
				if (fieldName.equals(SHARD_FIELD) && ! includeShard) {
					continue;
				}
				SchemaField field = schema.getField(fieldName);
				if (field == null || ! field.stored()) {
					continue;
				}

				Object value = doc.getFieldValue(fieldName);
				for (CopyField cp : schema.getCopyFieldsList(fieldName)) {
					addConvertedFieldValue(superDoc, value, cp.getDestination());
				}
				
				if (fieldName.equals(uniqueKeyField)) {
					// the [djoin] field value is [shard]:[id]:_version_
					if (includeShardId) {
						String shard = (String)doc.getFieldValue(SHARD_FIELD);
						String version = doc.getFieldValue(VERSION_FIELD).toString();
						addFieldValue(superDoc, shard + ":" + value + ":" + version, null);
					}
				} else {
					addConvertedFieldValue(superDoc, value, field);
				}
			}
		}
		feds.add(superDoc);
	}*/
	
	// for now, just add the numFounds for each shard to the results...
	/*NamedList details = new NamedList();
	rb.rsp.getValues().add("federated", details);
	Map<String, Long> numFounds = (Map<String, Long>)rb.req.getContext().get(COMPONENT_NAME + "numFounds");
	details.add("numFounds", numFounds);*/

	// ... and the size of joinIds
	/*Set<Object> joinIds = (Set<Object>)rb.req.getContext().get(COMPONENT_NAME + "joinIds");
	details.add("joinIds", joinIds);
	SolrDocumentList docs = (SolrDocumentList)rb.rsp.getValues().get("response");
	//docs.setNumFound((long)joinIds.size());
	numFounds.put("total", (long)joinIds.size());*/
}
 

@Override
public void inform(SolrCore core) {
  core.registerResponseWriter(AnalyticsShardResponseWriter.NAME, new AnalyticsShardResponseWriter());
  indexSchema = core.getLatestSchema();
  AnalyticsRequestParser.init();
}