/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.update;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.join.QueryBitSetProducer;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.handler.loader.XMLLoader;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.RefCounted;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public class AddBlockUpdateTest extends SolrTestCaseJ4 {
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  private static final String child = "child_s";
  private static final String parent = "parent_s";
  private static final String type = "type_s";

  private final static AtomicInteger counter = new AtomicInteger();
  private static ExecutorService exe;
  private static boolean cachedMode;

  private static XMLInputFactory inputFactory;

  private RefCounted<SolrIndexSearcher> searcherRef;
  private SolrIndexSearcher _searcher;

  @Rule
  public ExpectedException thrown = ExpectedException.none();

  @BeforeClass
  public static void beforeClass() throws Exception {
    String oldCacheNamePropValue = System
        .getProperty("blockJoinParentFilterCache");
    System.setProperty("blockJoinParentFilterCache", (cachedMode = random()
        .nextBoolean()) ? "blockJoinParentFilterCache" : "don't cache");
    if (oldCacheNamePropValue != null) {
      System.setProperty("blockJoinParentFilterCache", oldCacheNamePropValue);
    }
    inputFactory = XMLInputFactory.newInstance();

    exe = // Executors.newSingleThreadExecutor();
    rarely() ? ExecutorUtil.newMDCAwareFixedThreadPool(atLeast(2), new SolrNamedThreadFactory("AddBlockUpdateTest")) : ExecutorUtil
        .newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("AddBlockUpdateTest"));

    counter.set(0);
    initCore("solrconfig.xml", "schema15.xml");
  }

  @Before
  public void prepare() {
    // assertU("<rollback/>");
    assertU(delQ("*:*"));
    assertU(commit("expungeDeletes", "true"));
  }

  private Document getDocument() throws ParserConfigurationException {
    DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
    javax.xml.parsers.DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
    return docBuilder.newDocument();
  }

  private SolrIndexSearcher getSearcher() {
    if (_searcher == null) {
      searcherRef = h.getCore().getSearcher();
      _searcher = searcherRef.get();
    }
    return _searcher;
  }

  @After
  public void cleanup() {
    if (searcherRef != null || _searcher != null) {
      searcherRef.decref();
      searcherRef = null;
      _searcher = null;
    }
  }

  @AfterClass
  public static void afterClass() throws Exception {
    if (null != exe) {
      exe.shutdownNow();
      exe = null;
    }
    inputFactory = null;
  }

  @Test
  public void testOverwrite() throws IOException{
    assertU(add(
      nest(doc("id","X", parent, "X"),
             doc(child,"a", "id", "66"),
             doc(child,"b", "id", "66"))));
    assertU(add(
      nest(doc("id","Y", parent, "Y"),
             doc(child,"a", "id", "66"),
             doc(child,"b", "id", "66"))));
    String overwritten = random().nextBoolean() ? "X": "Y";
    String dubbed = overwritten.equals("X") ? "Y":"X";

    assertU(add(
        nest(doc("id",overwritten, parent, overwritten),
               doc(child,"c","id", "66"),
               doc(child,"d","id", "66")), "overwrite", "true"));
    assertU(add(
        nest(doc("id",dubbed, parent, dubbed),
               doc(child,"c","id", "66"),
               doc(child,"d","id", "66")), "overwrite", "false"));

    assertU(commit());

    assertQ(req(parent+":"+overwritten, "//*[@numFound='1']"));
    assertQ(req(parent+":"+dubbed, "//*[@numFound='2']"));

    final SolrIndexSearcher searcher = getSearcher();
    assertSingleParentOf(searcher, one("ab"), dubbed);

    final TopDocs docs = searcher.search(join(one("cd")), 10);
    assertEquals(2, docs.totalHits.value);
    final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent)+
                        searcher.doc(docs.scoreDocs[1].doc).get(parent);
    assertTrue(pAct.contains(dubbed) && pAct.contains(overwritten) && pAct.length()==2);

    assertQ(req("id:66", "//*[@numFound='6']"));
    assertQ(req(child+":(a b)", "//*[@numFound='2']"));
    assertQ(req(child+":(c d)", "//*[@numFound='4']"));
  }

  private static XmlDoc nest(XmlDoc parent, XmlDoc ... children){
    XmlDoc xmlDoc = new XmlDoc();
    xmlDoc.xml = parent.xml.replace("</doc>",
        Arrays.toString(children).replaceAll("[\\[\\]]", "")+"</doc>");
    return xmlDoc;
  }

  @Test
  public void testBasics() throws Exception {
    List<Document> blocks = new ArrayList<>(Arrays.asList(
        block("abcD"),
        block("efgH"),
        merge(block("ijkL"), block("mnoP")),
        merge(block("qrsT"), block("uvwX")),
        block("Y"),
        block("Z")));

    Collections.shuffle(blocks, random());

    log.trace("{}", blocks);

    for (Future<Void> f : exe.invokeAll(callables(blocks))) {
      f.get(); // exceptions?
    }

    assertU(commit());

    final SolrIndexSearcher searcher = getSearcher();
    // final String resp = h.query(req("q","*:*", "sort","_docid_ asc", "rows",
    // "10000"));
    // log.trace(resp);
    int parentsNum = "DHLPTXYZ".length();
    assertQ(req(parent + ":[* TO *]"), "//*[@numFound='" + parentsNum + "']");
    assertQ(req(child + ":[* TO *]"), "//*[@numFound='"
        + (('z' - 'a' + 1) - parentsNum) + "']");
    assertQ(req("*:*"), "//*[@numFound='" + ('z' - 'a' + 1) + "']");
    assertSingleParentOf(searcher, one("abc"), "D");
    assertSingleParentOf(searcher, one("efg"), "H");
    assertSingleParentOf(searcher, one("ijk"), "L");
    assertSingleParentOf(searcher, one("mno"), "P");
    assertSingleParentOf(searcher, one("qrs"), "T");
    assertSingleParentOf(searcher, one("uvw"), "X");

    assertQ(req("q",child+":(a b c)", "sort","_docid_ asc"),
        "//*[@numFound='3']", // assert physical order of children
      "//doc[1]/arr[@name='child_s']/str[text()='a']",
      "//doc[2]/arr[@name='child_s']/str[text()='b']",
      "//doc[3]/arr[@name='child_s']/str[text()='c']");
  }

  @Test
  public void testExceptionThrown() throws Exception {
    final String abcD = getStringFromDocument(block("abcD"));
    log.info(abcD);
    assertBlockU(abcD);

    Document docToFail = getDocument();
    Element root = docToFail.createElement("add");
    docToFail.appendChild(root);
    Element doc1 = docToFail.createElement("doc");
    root.appendChild(doc1);
    attachField(docToFail, doc1, "id", id());
    attachField(docToFail, doc1, parent, "Y");
    attachField(docToFail, doc1, "sample_i", "notanumber/ignore_exception");
    Element subDoc1 = docToFail.createElement("doc");
    doc1.appendChild(subDoc1);
    attachField(docToFail, subDoc1, "id", id());
    attachField(docToFail, subDoc1, child, "x");
    Element doc2 = docToFail.createElement("doc");
    root.appendChild(doc2);
    attachField(docToFail, doc2, "id", id());
    attachField(docToFail, doc2, parent, "W");

    assertFailedBlockU(getStringFromDocument(docToFail));

    assertBlockU(getStringFromDocument(block("efgH")));
    assertBlockU(commit());

    final SolrIndexSearcher searcher = getSearcher();
    assertQ(req("q","*:*","indent","true", "fl","id,parent_s,child_s"), "//*[@numFound='" + "abcDefgH".length() + "']");
    assertSingleParentOf(searcher, one("abc"), "D");
    assertSingleParentOf(searcher, one("efg"), "H");

    assertQ(req(child + ":x"), "//*[@numFound='0']");
    assertQ(req(parent + ":Y"), "//*[@numFound='0']");
    assertQ(req(parent + ":W"), "//*[@numFound='0']");
  }

  @Test
  public void testExceptionThrownChildDocWAnonymousChildren() throws Exception {
    SolrInputDocument document1 = sdoc("id", id(), parent, "X",
        "child1_s", sdoc("id", id(), "child_s", "y"),
        "child2_s", sdoc("id", id(), "child_s", "z"));

    SolrInputDocument exceptionChildDoc = (SolrInputDocument) document1.get("child1_s").getValue();
    addChildren("child", exceptionChildDoc, 0, false);

    thrown.expect(SolrException.class);
    final String expectedMessage = "Anonymous child docs can only hang from others or the root";
    thrown.expectMessage(expectedMessage);
    indexSolrInputDocumentsDirectly(document1);
  }

  @Test
  public void testSolrNestedFieldsList() throws Exception {
    final String id1 = id();
    List<SolrInputDocument> children1 = Arrays.asList(sdoc("id", id(), child, "y"), sdoc("id", id(), child, "z"));

    SolrInputDocument document1 = sdoc("id", id1, parent, "X",
        "children", children1);

    final String id2 = id();
    List<SolrInputDocument> children2 = Arrays.asList(sdoc("id", id(), child, "b"), sdoc("id", id(), child, "c"));

    SolrInputDocument document2 = sdoc("id", id2, parent, "A",
        "children", children2);

    indexSolrInputDocumentsDirectly(document1, document2);

    final SolrIndexSearcher searcher = getSearcher();
    assertJQ(req("q","*:*",
        "fl","*",
        "sort","id asc",
        "wt","json"),
        "/response/numFound==" + "XyzAbc".length());
    assertJQ(req("q",parent+":" + document2.getFieldValue(parent),
        "fl","*",
        "sort","id asc",
        "wt","json"),
        "/response/docs/[0]/id=='" + document2.getFieldValue("id") + "'");
    assertQ(req("q",child+":(y z b c)", "sort","_docid_ asc"),
        "//*[@numFound='" + "yzbc".length() + "']", // assert physical order of children
        "//doc[1]/arr[@name='child_s']/str[text()='y']",
        "//doc[2]/arr[@name='child_s']/str[text()='z']",
        "//doc[3]/arr[@name='child_s']/str[text()='b']",
        "//doc[4]/arr[@name='child_s']/str[text()='c']");
    assertSingleParentOf(searcher, one("bc"), "A");
    assertSingleParentOf(searcher, one("yz"), "X");
  }

  @Test
  public void testSolrNestedFieldsSingleVal() throws Exception {
    SolrInputDocument document1 = sdoc("id", id(), parent, "X",
        "child1_s", sdoc("id", id(), "child_s", "y"),
        "child2_s", sdoc("id", id(), "child_s", "z"));

    SolrInputDocument document2 = sdoc("id", id(), parent, "A",
        "child1_s", sdoc("id", id(), "child_s", "b"),
        "child2_s", sdoc("id", id(), "child_s", "c"));

    indexSolrInputDocumentsDirectly(document1, document2);

    final SolrIndexSearcher searcher = getSearcher();
    assertJQ(req("q","*:*",
        "fl","*",
        "sort","id asc",
        "wt","json"),
        "/response/numFound==" + "XyzAbc".length());
    assertJQ(req("q",parent+":" + document2.getFieldValue(parent),
        "fl","*",
        "sort","id asc",
        "wt","json"),
        "/response/docs/[0]/id=='" + document2.getFieldValue("id") + "'");
    assertQ(req("q",child+":(y z b c)", "sort","_docid_ asc"),
        "//*[@numFound='" + "yzbc".length() + "']", // assert physical order of children
        "//doc[1]/arr[@name='child_s']/str[text()='y']",
        "//doc[2]/arr[@name='child_s']/str[text()='z']",
        "//doc[3]/arr[@name='child_s']/str[text()='b']",
        "//doc[4]/arr[@name='child_s']/str[text()='c']");
    assertSingleParentOf(searcher, one("bc"), "A");
    assertSingleParentOf(searcher, one("yz"), "X");
  }

  @SuppressWarnings("serial")
  @Test
  public void testSolrJXML() throws Exception {
    UpdateRequest req = new UpdateRequest();

    List<SolrInputDocument> docs = new ArrayList<>();

    SolrInputDocument document1 = new SolrInputDocument() {
      {
        final String id = id();
        addField("id", id);
        addField("parent_s", "X");

        ArrayList<SolrInputDocument> ch1 = new ArrayList<>(
            Arrays.asList(new SolrInputDocument() {
              {
                addField("id", id());
                addField("child_s", "y");
              }
            }, new SolrInputDocument() {
              {
                addField("id", id());
                addField("child_s", "z");
              }
            }));

        Collections.shuffle(ch1, random());
        addChildDocuments(ch1);
      }
    };

    SolrInputDocument document2 = new SolrInputDocument() {
      {
        final String id = id();
        addField("id", id);
        addField("parent_s", "A");
        addChildDocument(new SolrInputDocument() {
          {
            addField("id", id());
            addField("child_s", "b");
          }
        });
        addChildDocument(new SolrInputDocument() {
          {
            addField("id", id());
            addField("child_s", "c");
          }
        });
      }
    };

    docs.add(document1);
    docs.add(document2);

    Collections.shuffle(docs, random());
    req.add(docs);

    RequestWriter requestWriter = new RequestWriter();
    OutputStream os = new ByteArrayOutputStream();
    requestWriter.write(req, os);
    assertBlockU(os.toString());
    assertU(commit());

    assertJQ(req("q","*:*",
        "fl","*",
        "sort","id asc",
        "wt","json"),
        "/response/numFound==" + 6);
    final SolrIndexSearcher searcher = getSearcher();
    assertSingleParentOf(searcher, one("yz"), "X");
    assertSingleParentOf(searcher, one("bc"), "A");
  }
  //This is the same as testSolrJXML above but uses the XMLLoader
  // to illustrate the structure of the XML documents
  @Test
  public void testXML() throws IOException, XMLStreamException {
    UpdateRequest req = new UpdateRequest();

    List<SolrInputDocument> docs = new ArrayList<>();

    String xml_doc1 =
    "<doc >" +
      "  <field name=\"id\">1</field>" +
      "  <field name=\"parent_s\">X</field>" +
         "<doc>  " +
         "  <field name=\"id\" >2</field>" +
         "  <field name=\"child_s\">y</field>" +
         "</doc>"+
         "<doc>  " +
         "  <field name=\"id\" >3</field>" +
         "  <field name=\"child_s\">z</field>" +
         "</doc>"+
    "</doc>";

    String xml_doc2 =
        "<doc >" +
          "  <field name=\"id\">4</field>" +
          "  <field name=\"parent_s\">A</field>" +
             "<doc>  " +
             "  <field name=\"id\" >5</field>" +
             "  <field name=\"child_s\">b</field>" +
             "</doc>"+
             "<doc>  " +
             "  <field name=\"id\" >6</field>" +
             "  <field name=\"child_s\">c</field>" +
             "</doc>"+
        "</doc>";


    XMLStreamReader parser =
      inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) );
    parser.next(); // read the START document...
    //null for the processor is all right here
    XMLLoader loader = new XMLLoader();
    SolrInputDocument document1 = loader.readDoc( parser );

    XMLStreamReader parser2 =
        inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) );
      parser2.next(); // read the START document...
      //null for the processor is all right here
      //XMLLoader loader = new XMLLoader();
      SolrInputDocument document2 = loader.readDoc( parser2 );

    docs.add(document1);
    docs.add(document2);

    Collections.shuffle(docs, random());
    req.add(docs);

    RequestWriter requestWriter = new RequestWriter();
    OutputStream os = new ByteArrayOutputStream();
    requestWriter.write(req, os);
    assertBlockU(os.toString());
    assertU(commit());

    final SolrIndexSearcher searcher = getSearcher();
    assertSingleParentOf(searcher, one("yz"), "X");
    assertSingleParentOf(searcher, one("bc"), "A");
  }

  @Test
  public void testXMLMultiLevelLabeledChildren() throws XMLStreamException {
    String xml_doc1 =
        "<doc >" +
            "  <field name=\"id\">1</field>" +
            "  <field name=\"empty_s\"></field>" +
            "  <field name=\"parent_s\">X</field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >2</field>" +
            "      <field name=\"child_s\">y</field>" +
            "    </doc>" +
            "    <doc>  " +
            "      <field name=\"id\" >3</field>" +
            "      <field name=\"child_s\">z</field>" +
            "    </doc>" +
            "  </field> " +
            "</doc>";

    String xml_doc2 =
        "<doc >" +
            "  <field name=\"id\">4</field>" +
            "  <field name=\"parent_s\">A</field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >5</field>" +
            "      <field name=\"child_s\">b</field>" +
            "      <field name=\"grandChild\">" +
            "        <doc>  " +
            "          <field name=\"id\" >7</field>" +
            "          <field name=\"child_s\">d</field>" +
            "        </doc>" +
            "      </field>" +
            "    </doc>" +
            "  </field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >6</field>" +
            "      <field name=\"child_s\">c</field>" +
            "    </doc>" +
            "  </field> " +
            "</doc>";

    XMLStreamReader parser =
        inputFactory.createXMLStreamReader(new StringReader(xml_doc1));
    parser.next(); // read the START document...
    //null for the processor is all right here
    XMLLoader loader = new XMLLoader();
    SolrInputDocument document1 = loader.readDoc(parser);

    XMLStreamReader parser2 =
        inputFactory.createXMLStreamReader(new StringReader(xml_doc2));
    parser2.next(); // read the START document...
    //null for the processor is all right here
    //XMLLoader loader = new XMLLoader();
    SolrInputDocument document2 = loader.readDoc(parser2);

    assertFalse(document1.hasChildDocuments());
    assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
        sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());

    assertFalse(document2.hasChildDocuments());
    assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
        sdocs(sdoc("id", "5", "child_s", "b", "grandChild", Collections.singleton(sdoc("id", "7", "child_s", "d"))),
            sdoc("id", "6", "child_s", "c"))).toString());
  }

  @Test
  public void testXMLLabeledChildren() throws IOException, XMLStreamException {
    UpdateRequest req = new UpdateRequest();

    List<SolrInputDocument> docs = new ArrayList<>();

    String xml_doc1 =
        "<doc >" +
            "  <field name=\"id\">1</field>" +
            "  <field name=\"empty_s\"></field>" +
            "  <field name=\"parent_s\">X</field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >2</field>" +
            "      <field name=\"child_s\">y</field>" +
            "    </doc>"+
            "    <doc>  " +
            "      <field name=\"id\" >3</field>" +
            "      <field name=\"child_s\">z</field>" +
            "    </doc>" +
            "  </field> " +
            "</doc>";

    String xml_doc2 =
        "<doc >" +
            "  <field name=\"id\">4</field>" +
            "  <field name=\"parent_s\">A</field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >5</field>" +
            "      <field name=\"child_s\">b</field>" +
            "    </doc>"+
            "  </field>" +
            "  <field name=\"test\">" +
            "    <doc>  " +
            "      <field name=\"id\" >6</field>" +
            "      <field name=\"child_s\">c</field>" +
            "    </doc>" +
            "  </field> " +
            "</doc>";

    XMLStreamReader parser =
        inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) );
    parser.next(); // read the START document...
    //null for the processor is all right here
    XMLLoader loader = new XMLLoader();
    SolrInputDocument document1 = loader.readDoc( parser );

    XMLStreamReader parser2 =
        inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) );
    parser2.next(); // read the START document...
    //null for the processor is all right here
    //XMLLoader loader = new XMLLoader();
    SolrInputDocument document2 = loader.readDoc( parser2 );

    assertFalse(document1.hasChildDocuments());
    assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
        sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());

    assertFalse(document2.hasChildDocuments());
    assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
        sdocs(sdoc("id", "5", "child_s", "b"), sdoc("id", "6", "child_s", "c"))).toString());

    docs.add(document1);
    docs.add(document2);

    Collections.shuffle(docs, random());
    req.add(docs);

    RequestWriter requestWriter = new RequestWriter();
    OutputStream os = new ByteArrayOutputStream();
    requestWriter.write(req, os);
    assertBlockU(os.toString());
    assertU(commit());

    final SolrIndexSearcher searcher = getSearcher();
    assertSingleParentOf(searcher, one("yz"), "X");
    assertSingleParentOf(searcher, one("bc"), "A");
  }

  @Test
  public void testJavaBinCodecNestedRelation() throws IOException {
    SolrInputDocument topDocument = new SolrInputDocument();
    topDocument.addField("parent_f1", "v1");
    topDocument.addField("parent_f2", "v2");

    int childsNum = atLeast(10);
    Map<String, SolrInputDocument> children = new HashMap<>(childsNum);
    for(int i = 0; i < childsNum; ++i) {
      SolrInputDocument child = new SolrInputDocument();
      child.addField("key", (i + 5) * atLeast(4));
      String childKey = String.format(Locale.ROOT, "child%d", i);
      topDocument.addField(childKey, child);
      children.put(childKey, child);
    }

    ByteArrayOutputStream os = new ByteArrayOutputStream();
    try (JavaBinCodec jbc = new JavaBinCodec()) {
      jbc.marshal(topDocument, os);
    }
    byte[] buffer = os.toByteArray();
    //now read the Object back
    SolrInputDocument result;
    try (JavaBinCodec jbc = new JavaBinCodec(); InputStream is = new ByteArrayInputStream(buffer)) {
      result = (SolrInputDocument) jbc.unmarshal(is);
    }

    assertTrue(compareSolrInputDocument(topDocument, result));
  }


  @Test
  public void testJavaBinCodec() throws IOException { //actually this test must be in other test class
    SolrInputDocument topDocument = new SolrInputDocument();
    topDocument.addField("parent_f1", "v1");
    topDocument.addField("parent_f2", "v2");

    int childsNum = atLeast(10);
    for (int index = 0; index < childsNum; ++index) {
      addChildren("child", topDocument, index, false);
    }

    ByteArrayOutputStream os = new ByteArrayOutputStream();
    try (JavaBinCodec jbc = new JavaBinCodec()) {
      jbc.marshal(topDocument, os);
    }
    byte[] buffer = os.toByteArray();
    //now read the Object back
    SolrInputDocument result;
    try (JavaBinCodec jbc = new JavaBinCodec(); InputStream is = new ByteArrayInputStream(buffer)) {
      result = (SolrInputDocument) jbc.unmarshal(is);
    }
    assertEquals(2, result.size());
    assertEquals("v1", result.getFieldValue("parent_f1"));
    assertEquals("v2", result.getFieldValue("parent_f2"));

    List<SolrInputDocument> resultChilds = result.getChildDocuments();
    int resultChildsSize = resultChilds == null ? 0 : resultChilds.size();
    assertEquals(childsNum, resultChildsSize);

    for (int childIndex = 0; childIndex < childsNum; ++childIndex) {
      SolrInputDocument child = resultChilds.get(childIndex);
      for (int fieldNum = 0; fieldNum < childIndex; ++fieldNum) {
        assertEquals(childIndex + "value" + fieldNum, child.getFieldValue(childIndex + "child" + fieldNum));
      }

      List<SolrInputDocument> grandChilds = child.getChildDocuments();
      int grandChildsSize = grandChilds == null ? 0 : grandChilds.size();

      assertEquals(childIndex * 2, grandChildsSize);
      for (int grandIndex = 0; grandIndex < childIndex * 2; ++grandIndex) {
        SolrInputDocument grandChild = grandChilds.get(grandIndex);
        assertFalse(grandChild.hasChildDocuments());
        for (int fieldNum = 0; fieldNum < grandIndex; ++fieldNum) {
          assertEquals(grandIndex + "value" + fieldNum, grandChild.getFieldValue(grandIndex + "grand" + fieldNum));
        }
      }
    }
  }

  private void addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel) {
    SolrInputDocument childDocument = new SolrInputDocument();
    for (int index = 0; index < childIndex; ++index) {
      childDocument.addField(childIndex + prefix + index, childIndex + "value"+ index);
    }

    if (!lastLevel) {
      for (int i = 0; i < childIndex * 2; ++i) {
        addChildren("grand", childDocument, i, true);
      }
    }
    topDocument.addChildDocument(childDocument);
  }

  /**
   * on the given abcD it generates one parent doc, taking D from the tail and
   * two subdocs relaitons ab and c uniq ids are supplied also
   *
   * <pre>
   * {@code
   * <add>
   *  <doc>
   *    <field name="parent_s">D</field>
   *    <doc>
   *        <field name="child_s">a</field>
   *        <field name="type_s">1</field>
   *    </doc>
   *    <doc>
   *        <field name="child_s">b</field>
   *        <field name="type_s">1</field>
   *    </doc>
   *    <doc>
   *        <field name="child_s">c</field>
   *        <field name="type_s">2</field>
   *    </doc>
   *  </doc>
   * </add>
   * }
   * </pre>
   * */
  private Document block(String string) throws ParserConfigurationException {
    Document document = getDocument();
    Element root = document.createElement("add");
    document.appendChild(root);
    Element doc = document.createElement("doc");
    root.appendChild(doc);

    if (string.length() > 0) {
      // last character is a top parent
      attachField(document, doc, parent,
          String.valueOf(string.charAt(string.length() - 1)));
      attachField(document, doc, "id", id());

      // add subdocs
      int type = 1;
      for (int i = 0; i < string.length() - 1; i += 2) {
        String relation = string.substring(i,
            Math.min(i + 2, string.length() - 1));
        attachSubDocs(document, doc, relation, type);
        type++;
      }
    }

    return document;
  }

  private void attachSubDocs(Document document, Element parent, String relation, int typeValue) {
    for (int j = 0; j < relation.length(); j++) {
      Element doc = document.createElement("doc");
      parent.appendChild(doc);
      attachField(document, doc, child, String.valueOf(relation.charAt(j)));
      attachField(document, doc, "id", id());
      attachField(document, doc, type, String.valueOf(typeValue));
    }
  }

  private void indexSolrInputDocumentsDirectly(SolrInputDocument ... docs) throws IOException {
    SolrQueryRequest coreReq = new LocalSolrQueryRequest(h.getCore(), new ModifiableSolrParams());
    AddUpdateCommand updateCmd = new AddUpdateCommand(coreReq);
    for (SolrInputDocument doc: docs) {
      updateCmd.solrDoc = doc;
      h.getCore().getUpdateHandler().addDoc(updateCmd);
      updateCmd.clear();
    }
    assertU(commit());
  }

  /**
   * Merges two documents like
   *
   * <pre>
   * {@code <add>...</add> + <add>...</add> = <add>... + ...</add>}
   * </pre>
   *
   * @param doc1
   *          first document
   * @param doc2
   *          second document
   * @return merged document
   */
  private Document merge(Document doc1, Document doc2) {
    NodeList doc2ChildNodes = doc2.getDocumentElement().getChildNodes();
    for(int i = 0; i < doc2ChildNodes.getLength(); i++) {
      Node doc2ChildNode = doc2ChildNodes.item(i);
      doc1.getDocumentElement().appendChild(doc1.importNode(doc2ChildNode, true));
      doc2.getDocumentElement().removeChild(doc2ChildNode);
    }

    return doc1;
  }

  private void attachField(Document document, Element root, String fieldName, String value) {
    Element field = document.createElement("field");
    field.setAttribute("name", fieldName);
    field.setTextContent(value);
    root.appendChild(field);
  }

  private static String id() {
    return "" + counter.incrementAndGet();
  }

  private String one(String string) {
    return "" + string.charAt(random().nextInt(string.length()));
  }

  protected void assertSingleParentOf(final SolrIndexSearcher searcher,
      final String childTerm, String parentExp) throws IOException {
    final TopDocs docs = searcher.search(join(childTerm), 10);
    assertEquals(1, docs.totalHits.value);
    final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent);
    assertEquals(parentExp, pAct);
  }

  protected ToParentBlockJoinQuery join(final String childTerm) {
    return new ToParentBlockJoinQuery(
        new TermQuery(new Term(child, childTerm)), new QueryBitSetProducer(
            new TermRangeQuery(parent, null, null, false, false)), ScoreMode.None);
  }

  private Collection<? extends Callable<Void>> callables(List<Document> blocks) {
    final List<Callable<Void>> rez = new ArrayList<>();
    for (Document block : blocks) {
      final String msg = getStringFromDocument(block);
      if (msg.length() > 0) {
        rez.add(() -> {
          assertBlockU(msg);
          return null;
        });
        if (rarely()) {
          rez.add(() -> {
            assertBlockU(commit());
            return null;
          });
        }
      }
    }
    return rez;
  }

  private void assertBlockU(final String msg) {
    assertBlockU(msg, "0");
  }

  private void assertFailedBlockU(final String msg) {
    expectThrows(Exception.class, () -> assertBlockU(msg, "1"));
  }

  private void assertBlockU(final String msg, String expected) {
    try {
      String res = h.checkUpdateStatus(msg, expected);
      if (res != null) {
        fail("update was not successful: " + res + " expected: " + expected);
      }
    } catch (SAXException e) {
      throw new RuntimeException("Invalid XML", e);
    }
  }

  public static String getStringFromDocument(Document doc) {
    try (StringWriter writer = new StringWriter()){
      TransformerFactory tf = TransformerFactory.newInstance();
      Transformer transformer = tf.newTransformer();
      transformer.transform(new DOMSource(doc), new StreamResult(writer));
      return writer.toString();
    } catch (TransformerException | IOException e) {
      throw new IllegalStateException(e);
    }
  }
}