Java Code Examples for org.apache.solr.core.SolrCore#getUpdateProcessingChain()

protected void processCommit(final String chain) throws IOException {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();

  SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());

  CommitUpdateCommand cmd = new CommitUpdateCommand(req,false);
  UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
  try {
    processor.processCommit(cmd);
  } finally {
    req.close();
  }
}
 

protected void processDeleteById(final String chain, String id) throws IOException {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();

  SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());

  DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
  cmd.setId(id);
  UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
  try {
    processor.processDelete(cmd);
  } finally {
    req.close();
  }
}
 

protected void finish(final String chain) throws IOException {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();
  SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());

  UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
  try {
    processor.finish();
  } finally {
    IOUtils.closeQuietly(processor);
    req.close();
  }
}
 

@Override
public void inform(SolrCore core) {
  this.core = core;
  
  if (null == core.getLatestSchema().getFieldTypeNoEx(expireField)) {
    // TODO: check for managed schema and auto-add as a date field?
    throw confErr(EXP_FIELD_NAME_CONF + " does not exist in schema: " + expireField);
  }

  if (0 < deletePeriodSeconds) {
    // validate that we have a chain we can work with
    try {
      Object ignored = core.getUpdateProcessingChain(deleteChainName);
    } catch (SolrException e) {
      throw confErr(DEL_CHAIN_NAME_CONF + " does not exist: " + deleteChainName, e);
    }
    // schedule recurring deletion
    initDeleteExpiredDocsScheduler(core);
  }
}
 

@Test
public void testDupeAllFieldsDetection() throws Exception {
  
  this.chain = "dedupe-allfields";
  
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain chained = core.getUpdateProcessingChain(this.chain);
  SignatureUpdateProcessorFactory factory = ((SignatureUpdateProcessorFactory) chained.getProcessors().get(0));
  factory.setEnabled(true);
  assertNotNull(chained);

  addDoc(adoc("v_t", "Hello Dude man!"));
  addDoc(adoc("v_t", "Hello Dude man!", "name", "name1'"));
  addDoc(adoc("v_t", "Hello Dude man!", "name", "name2'"));

  addDoc(commit());
  
  checkNumDocs(3);

  factory.setEnabled(false);
}
 

SolrQueryResponse processCommit(final String chain, boolean optimize, Boolean commitEndPoint) throws IOException {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();
  SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());

  if (commitEndPoint != null) {
    ((ModifiableSolrParams)req.getParams()).set(
        DistributedUpdateProcessor.COMMIT_END_POINT, commitEndPoint.booleanValue());
  }

  try {
    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp));
    CommitUpdateCommand cmd = new CommitUpdateCommand(req, false);
    cmd.optimize = optimize;
    UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
    processor.processCommit(cmd);
  } finally {
    SolrRequestInfo.clearRequestInfo();
    req.close();
  }
  return rsp;
}
 

/**
 * Runs a document through the specified chain, and returns the final
 * document used when the chain is completed (NOTE: some chains may
 * modify the document in place
 */
protected SolrInputDocument processAdd(final String chain,
                                       final SolrParams requestParams,
                                       final SolrInputDocument docIn)
  throws IOException {

  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();

  SolrQueryRequest req = new LocalSolrQueryRequest(core, requestParams);
  try {
    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
    AddUpdateCommand cmd = new AddUpdateCommand(req);
    cmd.solrDoc = docIn;

    UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
    if (null != processor) {
      // test chain might be empty or short circuited.
      processor.processAdd(cmd);
    }

    return cmd.solrDoc;
  } finally {
    SolrRequestInfo.clearRequestInfo();
    req.close();
  }
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  initCore("solrconfig-languageidentifier.xml", "schema.xml", getFile("langid/solr").getAbsolutePath());
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("lang_id_tika");
  assertNotNull(chained);
  chained = core.getUpdateProcessingChain("lang_id_lang_detect");
  assertNotNull(chained);
  chained = core.getUpdateProcessingChain("lang_id_opennlp");
  assertNotNull(chained);
}
 

public void testSingleScript() throws Exception {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("single-script");
  final StatelessScriptUpdateProcessorFactory factory = ((StatelessScriptUpdateProcessorFactory) chained.getProcessors().get(0));
  final List<String> functionMessages = new ArrayList<>();
  factory.setScriptEngineCustomizer(new ScriptEngineCustomizer() {
    @Override
    public void customize(ScriptEngine engine) {
      engine.put("functionMessages", functionMessages);
    }
  });
  assertNotNull(chained);

  SolrInputDocument d = processAdd("single-script",
      doc(f("id", "1"),
          f("name", " foo "),
          f("subject", "bar")));

  processCommit("run-no-scripts");

  assertQ("couldn't find doc by id",
          req("q","id:1")
          , "//result[@numFound=1]");

  processDeleteById("single-script","1");
  processCommit("single-script");
  
  assertQ("found deleted doc",
          req("q","id:1")
          , "//result[@numFound=0]");


  assertEquals(3, functionMessages.size());

  assertTrue(functionMessages.contains("processAdd0"));
  assertTrue(functionMessages.contains("processDelete0"));
  assertTrue(functionMessages.contains("processCommit0"));

}
 

public void testConfiguration() throws Exception 
{
  SolrCore core = h.getCore();

  // make sure it loaded the factories
  UpdateRequestProcessorChain chained = core.getUpdateProcessingChain( "standard" );
  
  // Make sure it got 3 items (4 configured, 1 is enable=false)
  assertEquals("wrong number of (enabled) factories in chain",
               3, chained.getProcessors().size() );

  // first one should be log, and it should be configured properly
  UpdateRequestProcessorFactory first = chained.getProcessors().get(0);
  assertEquals("wrong factory at front of chain",
               LogUpdateProcessorFactory.class, first.getClass());
  LogUpdateProcessorFactory log = (LogUpdateProcessorFactory)first;
  assertEquals("wrong config for LogUpdateProcessorFactory.maxNumToLog",
               100, log.maxNumToLog );
  assertEquals("wrong config for LogUpdateProcessorFactory.slowUpdateThresholdMillis",
               2000, log.slowUpdateThresholdMillis);


  UpdateRequestProcessorChain custom = core.getUpdateProcessingChain( null );
  CustomUpdateRequestProcessorFactory link = (CustomUpdateRequestProcessorFactory) custom.getProcessors().get(0);
  
  assertEquals( custom, core.getUpdateProcessingChain( "" ) );
  assertEquals( custom, core.getUpdateProcessingChain( "custom" ) );
  
  // Make sure the NamedListArgs got through ok
  assertEquals( "{name={n8=88, n9=99}}", link.args.toString() );
}
 

SolrInputDocument processAdd(final String chain,
                             final SolrInputDocument docIn, SolrParams params)
    throws IOException {

  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = chain == null ?
      core.getUpdateProcessorChain(params) :
      core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();

  SolrQueryRequest req = new LocalSolrQueryRequest
      (core, params);
  try {
    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp));
    AddUpdateCommand cmd = new AddUpdateCommand(req);
    cmd.solrDoc = docIn;

    UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
    processor.processAdd(cmd);

    return cmd.solrDoc;
  } finally {
    SolrRequestInfo.clearRequestInfo();
    req.close();
  }
}
 

/**
 * Runs a document through the specified chain, and returns the final 
 * document used when the chain is completed (NOTE: some chains may 
 * modify the document in place
 */
SolrInputDocument processAdd(final String chain, 
                             final SolrInputDocument docIn) 
  throws IOException {

  SolrCore core = h.getCore();
  UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
  assertNotNull("No Chain named: " + chain, pc);

  SolrQueryResponse rsp = new SolrQueryResponse();

  SolrQueryRequest req = new LocalSolrQueryRequest
    (core, new ModifiableSolrParams());
  try {
    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req,rsp));
    AddUpdateCommand cmd = new AddUpdateCommand(req);
    cmd.solrDoc = docIn;

    UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
    processor.processAdd(cmd);

    return cmd.solrDoc;
  } finally {
    SolrRequestInfo.clearRequestInfo();
    req.close();
  }
}
 

public void testMultipleScripts() throws Exception {
  SolrCore core = h.getCore();

  for (final String chain : new String[] {"dual-scripts-arr", 
                                          "dual-scripts-strs"}) {
  
    UpdateRequestProcessorChain chained = core.getUpdateProcessingChain(chain);
    final StatelessScriptUpdateProcessorFactory factory = 
      ((StatelessScriptUpdateProcessorFactory) chained.getProcessors().get(0));
    final List<String> functionMessages = new ArrayList<>();
    ScriptEngineCustomizer customizer = new ScriptEngineCustomizer() {
        @Override
        public void customize(ScriptEngine engine) {
          engine.put("functionMessages", functionMessages);
        }
      };
    factory.setScriptEngineCustomizer(customizer);
    assertNotNull(chained);

    SolrInputDocument d = processAdd(chain,
                                     doc(f("id", "2"),
                                         f("name", " foo "),
                                         f("subject", "bar")));
    
    assertEquals(chain + " didn't add Double field", 
                 42.3d, d.getFieldValue("script_added_d"));
    assertEquals(chain + " didn't add integer field",
        42, d.getFieldValue("script_added_i"));
    
    processCommit("run-no-scripts");

    assertQ(chain + ": couldn't find doc by id",
            req("q","id:2")
            , "//result[@numFound=1]");

    processDeleteById(chain, "2");
    processCommit(chain);
    
    assertEquals(chain, 6, functionMessages.size());
    assertTrue(chain, functionMessages.contains("processAdd0"));
    assertTrue(chain, functionMessages.contains("processAdd1"));
    assertTrue(chain + ": script order doesn't match conf order",
               functionMessages.indexOf("processAdd0") 
               < functionMessages.indexOf("processAdd1"));

    assertTrue(chain, functionMessages.contains("processDelete0"));
    assertTrue(chain, functionMessages.contains("processDelete1"));
    assertTrue(chain + ": script order doesn't match conf order",
               functionMessages.indexOf("processDelete0") 
               < functionMessages.indexOf("processDelete1"));

    assertTrue(chain, functionMessages.contains("processCommit0"));
    assertTrue(chain, functionMessages.contains("processCommit1"));
    assertTrue(chain + ": script order doesn't match conf order",
               functionMessages.indexOf("processCommit0") 
               < functionMessages.indexOf("processCommit1"));

    finish(chain);
  
    assertEquals(chain, 8, functionMessages.size());

    assertTrue(chain, functionMessages.contains("finish0"));
    assertTrue(chain, functionMessages.contains("finish1"));
    assertTrue(chain + ": script order doesn't match conf order",
               functionMessages.indexOf("finish0") 
               < functionMessages.indexOf("finish1"));

    assertQ(chain + ": found deleted doc",
            req("q","id:2")
            , "//result[@numFound=0]");
    
  }
}
 

public void testUpdateDistribChainSkipping() throws Exception {

    // a key part of this test is verifying that LogUpdateProcessor is found in all chains because it
    // is a @RunAlways processor -- but in order for that to work, we have to sanity check that the log
    // level is at least "INFO" otherwise the factory won't even produce a processor and all our assertions
    // are for nought.  (see LogUpdateProcessorFactory.getInstance)
    //
    // TODO: maybe create a new mock Processor w/ @RunAlways annot if folks feel requiring INFO is evil.
    assertTrue("Tests must be run with INFO level logging "+
               "otherwise LogUpdateProcessor isn't used and can't be tested.", log.isInfoEnabled());
    
    final int EXPECTED_CHAIN_LENGTH = 5;
    SolrCore core = h.getCore();
    for (final String name : Arrays.asList("distrib-chain-explicit",
                                           "distrib-chain-implicit",
                                           "distrib-chain-noop")) {

      UpdateRequestProcessor proc;
      List<UpdateRequestProcessor> procs;
      
      UpdateRequestProcessorChain chain = core.getUpdateProcessingChain(name);
      assertNotNull(name, chain);

      // either explicitly, or because of injection
      assertEquals(name + " factory chain length: " + chain.toString(), EXPECTED_CHAIN_LENGTH,
                   chain.getProcessors().size());

      // test a basic (non distrib) chain
      proc = chain.createProcessor(req(), new SolrQueryResponse());
      procs = procToList(proc);

      int expectedProcLen = EXPECTED_CHAIN_LENGTH;
      if ("distrib-chain-noop".equals(name)) { // NoOpDistributingUpdateProcessorFactory produces no processor
        expectedProcLen--;
      }
      if (procs.stream().anyMatch(p -> p.getClass().getSimpleName().equals("NestedUpdateProcessor"))) {
        expectedProcLen++; // NestedUpdate sneaks in via RunUpdate's Factory.
      }

      assertEquals(name + " procs size: " + procs.toString(), expectedProcLen, procs.size());
      
      // Custom comes first in all three of our chains
      assertTrue(name + " first processor isn't a CustomUpdateRequestProcessor: " + procs.toString(),
                 ( // compare them both just because i'm going insane and the more checks the better
                   proc instanceof CustomUpdateRequestProcessor
                   && procs.get(0) instanceof CustomUpdateRequestProcessor));

      // Log should always come second in our chain.
      assertNotNull(name + " proc.next is null", proc.next);
      assertNotNull(name + " second proc is null", procs.get(1));

      assertTrue(name + " second proc isn't LogUpdateProcessor: " + procs.toString(),
                 ( // compare them both just because i'm going insane and the more checks the better
                   proc.next instanceof LogUpdateProcessorFactory.LogUpdateProcessor
                   && procs.get(1) instanceof LogUpdateProcessorFactory.LogUpdateProcessor));

      // fetch the distributed version of this chain
      proc = chain.createProcessor(req(DISTRIB_UPDATE_PARAM, "NONE"), // just some non-blank value
                                   new SolrQueryResponse());
      procs = procToList(proc);
      assertNotNull(name + " (distrib) chain produced null proc", proc);
      assertFalse(name + " (distrib) procs is empty", procs.isEmpty());

      // for these 3 (distrib) chains, the first proc should always be LogUpdateProcessor
      assertTrue(name + " (distrib) first proc should be LogUpdateProcessor because of @RunAlways: "
                 + procs.toString(),
                 ( // compare them both just because i'm going insane and the more checks the better
                   proc instanceof LogUpdateProcessorFactory.LogUpdateProcessor
                   && procs.get(0) instanceof LogUpdateProcessorFactory.LogUpdateProcessor));

      // for these 3 (distrib) chains, the last proc should always be RunUpdateProcessor
      assertTrue(name + " (distrib) last processor isn't a RunUpdateProcessor: " + procs.toString(),
                 procs.get(procs.size()-1) instanceof RunUpdateProcessorFactory.RunUpdateProcessor );

      // either 1 proc was droped in distrib mode, or 1 for the "implicit" chain

      expectedProcLen = EXPECTED_CHAIN_LENGTH;
      expectedProcLen--; // -1 = all chains lose CustomUpdateRequestProcessorFactory
      if ("distrib-chain-explicit".equals(name) == false) {
        // -1 = distrib-chain-noop: NoOpDistributingUpdateProcessorFactory produces no processor
        // -1 = distrib-chain-implicit: does RemoveBlank before distrib
        expectedProcLen--;
      }
      if (procs.stream().anyMatch(p -> p.getClass().getSimpleName().equals("NestedUpdateProcessor"))) {
        expectedProcLen++; // NestedUpdate sneaks in via RunUpdate's Factory.
      }
      assertEquals(name + " (distrib) chain has wrong length: " + procs.toString(),
          expectedProcLen, procs.size());
    }

  }
 

@Test
public void testUniqFields() throws Exception {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain chained = core
    .getUpdateProcessingChain("uniq-fields");
  UniqFieldsUpdateProcessorFactory factory = ((UniqFieldsUpdateProcessorFactory) chained.getProcessors().get(0));
  assertNotNull(chained);

  addDoc(adoc("id", "1a", 
              "uniq", "value1", 
              "uniq", "value1", 
              "uniq", "value2"));
  addDoc(adoc("id", "2a", 
              "uniq2", "value1", 
              "uniq2", "value2", 
              "uniq2", "value1", 
              "uniq2", "value3", 
              "uniq", "value1", 
              "uniq", "value1"));
  addDoc(adoc("id", "1b", 
              "uniq3", "value1", 
              "uniq3", "value1"));
  addDoc(adoc("id", "1c", 
              "nouniq", "value1", 
              "nouniq", "value1", 
              "nouniq", "value2"));
  addDoc(adoc("id", "2c", 
              "nouniq", "value1", 
              "nouniq", "value1", 
              "nouniq", "value2", 
              "uniq2", "value1", 
              "uniq2", "value1"));

  assertU(commit());
  assertQ(req("id:1a"), "count(//*[@name='uniq']/*)=2",
      "//arr[@name='uniq']/str[1][.='value1']",
      "//arr[@name='uniq']/str[2][.='value2']");
  assertQ(req("id:2a"), "count(//*[@name='uniq2']/*)=3",
      "//arr[@name='uniq2']/str[1][.='value1']",
      "//arr[@name='uniq2']/str[2][.='value2']",
      "//arr[@name='uniq2']/str[3][.='value3']");
  assertQ(req("id:2a"), "count(//*[@name='uniq']/*)=1");
  assertQ(req("id:1b"), "count(//*[@name='uniq3'])=1");
  assertQ(req("id:1c"), "count(//*[@name='nouniq']/*)=3");
  assertQ(req("id:2c"), "count(//*[@name='nouniq']/*)=3");
  assertQ(req("id:2c"), "count(//*[@name='uniq2']/*)=1");

}
 

@Test
public void testDupeDetection() throws Exception {
  SolrCore core = h.getCore();
  UpdateRequestProcessorChain chained = core.getUpdateProcessingChain(
      "dedupe");
  SignatureUpdateProcessorFactory factory = ((SignatureUpdateProcessorFactory) chained.getProcessors().get(0));
  factory.setEnabled(true);
  assertNotNull(chained);

  addDoc(adoc("id", "1a", "v_t", "Hello Dude man!", "name", "ali babi'"));
  addDoc(adoc("id", "2a", "name", "ali babi", "v_t", "Hello Dude man . -"));

  addDoc(commit());

  addDoc(adoc("name", "ali babi'", "id", "3a", "v_t", "Hello Dude man!"));

  addDoc(commit());

  checkNumDocs(1);

  addDoc(adoc("id", "3b", "v_t", "Hello Dude man!", "t_field",
      "fake value galore"));

  addDoc(commit());

  checkNumDocs(2);

  assertU(adoc("id", "5a", "name", "ali babi", "v_t", "MMMMM"));

  addDoc(delI("5a"));

  addDoc(adoc("id", "5a", "name", "ali babi", "v_t", "MMMMM"));

  addDoc(commit());

  checkNumDocs(3);

  addDoc(adoc("id", "same", "name", "baryy white", "v_t", "random1"));
  addDoc(adoc("id", "same", "name", "bishop black", "v_t", "random2"));

  addDoc(commit());

  checkNumDocs(4);
  factory.setEnabled(false);
}