Java Code Examples for org.apache.lucene.facet.taxonomy.TaxonomyReader#openIfChanged()

@Test
public void testOpenIfChangedAndRefCount() throws Exception {
  Directory dir = new ByteBuffersDirectory(); // no need for random directories here

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.commit();

  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoReader.incRef();
  assertEquals("wrong refCount", 2, taxoReader.getRefCount());

  taxoWriter.addCategory(new FacetLabel("a", "b"));
  taxoWriter.commit();
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
  assertNotNull(newtr);
  taxoReader.close();
  taxoReader = newtr;
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoWriter.close();
  taxoReader.close();
  dir.close();
}
 

private void openReaderIfChanges() throws IOException {
	DirectoryReader newDirectoryReader = DirectoryReader
			.openIfChanged(directoryReader, indexWriter, indexConfig.getIndexSettings().getApplyUncommittedDeletes());
	if (newDirectoryReader != null) {
		directoryReader = newDirectoryReader;
		QueryResultCache qrc = queryResultCache;
		if (qrc != null) {
			qrc.clear();
		}

	}

	DirectoryTaxonomyReader newone = TaxonomyReader.openIfChanged(taxoReader);
	if (newone != null) {
		taxoReader = newone;
	}
}
 

@Override
public Boolean call() throws Exception {
  if (indexReader == null) {
    indexReader = DirectoryReader.open(indexDir);
    lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
    taxoReader = new DirectoryTaxonomyReader(taxoDir);
  } else {
    // verify search index
    DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
    assertNotNull("should not have reached here if no changes were made to the index", newReader);
    long newGeneration = newReader.getIndexCommit().getGeneration();
    assertTrue("expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration, newGeneration > lastIndexGeneration);
    indexReader.close();
    indexReader = newReader;
    lastIndexGeneration = newGeneration;
    TestUtil.checkIndex(indexDir);
    
    // verify taxonomy index
    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
    if (newTaxoReader != null) {
      taxoReader.close();
      taxoReader = newTaxoReader;
    }
    TestUtil.checkIndex(taxoDir);
    
    // verify faceted search
    int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    FacetsCollector fc = new FacetsCollector();
    searcher.search(new MatchAllDocsQuery(), fc);
    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
    assertEquals(1, facets.getSpecificValue("A", Integer.toString(id, 16)).intValue());
    
    DrillDownQuery drillDown = new DrillDownQuery(config);
    drillDown.add("A", Integer.toString(id, 16));
    TopDocs docs = searcher.search(drillDown, 10);
    assertEquals(1, docs.totalHits.value);
  }
  return null;
}
 

@Test
public void testRecreateAndRefresh() throws Exception {
  // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
  // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
  // DirTaxoReader succeeding to refresh().
  try (Directory dir = newDirectory()) {

    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("a"));

    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);

    touchTaxo(taxoWriter, new FacetLabel("b"));

    TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));

    // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("c"));
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("d"));
    taxoWriter.close();

    newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
    taxoReader.close();
  }
}
 

@Test
public void testOpenIfChangedResult() throws Exception {
  Directory dir = null;
  DirectoryTaxonomyWriter ltw = null;
  DirectoryTaxonomyReader ltr = null;
  
  try {
    dir = newDirectory();
    ltw = new DirectoryTaxonomyWriter(dir);
    
    ltw.addCategory(new FacetLabel("a"));
    ltw.commit();
    
    ltr = new DirectoryTaxonomyReader(dir);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr));
    
    ltw.addCategory(new FacetLabel("b"));
    ltw.commit();
    
    DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr);
    assertNotNull("changes were committed", newtr);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(newtr));
    newtr.close();
  } finally {
    IOUtils.close(ltw, ltr, dir);
  }
}
 

@Test
public void testOpenIfChangedReuseAfterRecreate() throws Exception {
  // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
  FacetLabel cp_a = new FacetLabel("a");
  writer.addCategory(cp_a);
  writer.close();
  
  DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
  // fill r1's caches
  assertEquals(1, r1.getOrdinal(cp_a));
  assertEquals(cp_a, r1.getPath(1));
  
  // now recreate, add a different category
  writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
  FacetLabel cp_b = new FacetLabel("b");
  writer.addCategory(cp_b);
  writer.close();
  
  DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
  assertNotNull(r2);
  
  // fill r2's caches
  assertEquals(1, r2.getOrdinal(cp_b));
  assertEquals(cp_b, r2.getPath(1));
  
  // check that r1 doesn't see cp_b
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
  assertEquals(cp_a, r1.getPath(1));

  // check that r2 doesn't see cp_a
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
  assertEquals(cp_b, r2.getPath(1));

  r2.close();
  r1.close();
  dir.close();
}
 

@Test
public void testOpenIfChangedReuse() throws Exception {
  // test the reuse of data from the old DTR instance
  for (boolean nrt : new boolean[] {false, true}) {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));
    
    FacetLabel cp_b = new FacetLabel("b");
    writer.addCategory(cp_b);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);
    
    // add r2's categories to the caches
    assertEquals(2, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(2));
    
    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertNull(r1.getPath(2));
    
    r1.close();
    r2.close();
    writer.close();
    dir.close();
  }
}
 

private void doTestReadRecreatedTaxonomy(Random random, boolean closeReader) throws Exception {
  Directory dir = null;
  TaxonomyWriter tw = null;
  TaxonomyReader tr = null;
  
  // prepare a few categories
  int  n = 10;
  FacetLabel[] cp = new FacetLabel[n];
  for (int i=0; i<n; i++) {
    cp[i] = new FacetLabel("a", Integer.toString(i));
  }
  
  try {
    dir = newDirectory();
    
    tw = new DirectoryTaxonomyWriter(dir);
    tw.addCategory(new FacetLabel("a"));
    tw.close();
    
    tr = new DirectoryTaxonomyReader(dir);
    int baseNumCategories = tr.getSize();
    
    for (int i=0; i<n; i++) {
      int k = random.nextInt(n);
      tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
      for (int j = 0; j <= k; j++) {
        tw.addCategory(cp[j]);
      }
      tw.close();
      if (closeReader) {
        tr.close();
        tr = new DirectoryTaxonomyReader(dir);
      } else {
        TaxonomyReader newtr = TaxonomyReader.openIfChanged(tr);
        assertNotNull(newtr);
        tr.close();
        tr = newtr;
      }
      assertEquals("Wrong #categories in taxonomy (i="+i+", k="+k+")", baseNumCategories + 1 + k, tr.getSize());
    }
  } finally {
    IOUtils.close(tr, tw, dir);
  }
}
 

@Test
public void testOpenIfChangedManySegments() throws Exception {
  // test openIfChanged() when the taxonomy contains many segments
  Directory dir = newDirectory();
  
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
      IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
      lmp.setMergeFactor(2);
      return conf;
    }
  };
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  
  int numRounds = random().nextInt(10) + 10;
  int numCategories = 1; // one for root
  for (int i = 0; i < numRounds; i++) {
    int numCats = random().nextInt(4) + 1;
    for (int j = 0; j < numCats; j++) {
      writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j)));
    }
    numCategories += numCats + 1 /* one for round-parent */;
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
    assertNotNull(newtr);
    reader.close();
    reader = newtr;
    
    // assert categories
    assertEquals(numCategories, reader.getSize());
    int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i)));
    int[] parents = reader.getParallelTaxonomyArrays().parents();
    assertEquals(0, parents[roundOrdinal]); // round's parent is root
    for (int j = 0; j < numCats; j++) {
      int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j)));
      assertEquals(roundOrdinal, parents[ord]); // round's parent is root
    }
  }
  
  reader.close();
  writer.close();
  dir.close();
}
 

@Test
public void testOpenIfChangedMergedSegment() throws Exception {
  // test openIfChanged() when all index segments were merged - used to be
  // a bug in ParentArray, caught by testOpenIfChangedManySegments - only
  // this test is not random
  Directory dir = newDirectory();
  
  // hold onto IW to forceMerge
  // note how we don't close it, since DTW will close it.
  final IndexWriter iw = new IndexWriter(dir,
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setMergePolicy(new LogByteSizeMergePolicy()));
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriter openIndexWriter(Directory directory,
        IndexWriterConfig config) throws IOException {
      return iw;
    }
  };
  
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  assertEquals(1, reader.getSize());
  assertEquals(1, reader.getParallelTaxonomyArrays().parents().length);

  // add category and call forceMerge -- this should flush IW and merge segments down to 1
  // in ParentArray.initFromReader, this used to fail assuming there are no parents.
  writer.addCategory(new FacetLabel("1"));
  iw.forceMerge(1);
  
  // now calling openIfChanged should trip on the bug
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
  assertNotNull(newtr);
  reader.close();
  reader = newtr;
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
  
  reader.close();
  writer.close();
  dir.close();
}
 

@Test
public void testOpenIfChangedNoChangesButSegmentMerges() throws Exception {
  // test openIfChanged() when the taxonomy hasn't really changed, but segments
  // were merged. The NRT reader will be reopened, and ParentArray used to assert
  // that the new reader contains more ordinals than were given from the old
  // TaxReader version
  Directory dir = newDirectory();
  
  // hold onto IW to forceMerge
  // note how we don't close it, since DTW will close it.
  final IndexWriter iw = new IndexWriter(dir,
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setMergePolicy(new LogByteSizeMergePolicy()));
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriter openIndexWriter(Directory directory,
        IndexWriterConfig config) throws IOException {
      return iw;
    }
  };
  
  // add a category so that the following DTR open will cause a flush and 
  // a new segment will be created
  writer.addCategory(new FacetLabel("a"));
  
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);

  // merge all the segments so that NRT reader thinks there's a change 
  iw.forceMerge(1);
  
  // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
  assertNotNull(newtr);
  reader.close();
  reader = newtr;
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
  
  reader.close();
  writer.close();
  dir.close();
}
 

@Test
public void testOpenIfChangedReplaceTaxonomy() throws Exception {
  // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
  // only can work with NRT as well
  Directory src = newDirectory();
  DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
  FacetLabel cp_b = new FacetLabel("b");
  w.addCategory(cp_b);
  w.close();
  
  for (boolean nrt : new boolean[] {false, true}) {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));

    // now replace taxonomy
    writer.replaceTaxonomy(src);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);
    
    // fill r2's caches
    assertEquals(1, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(1));
    
    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertEquals(cp_a, r1.getPath(1));

    // check that r2 doesn't see cp_a
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
    assertEquals(cp_b, r2.getPath(1));

    r2.close();
    r1.close();
    writer.close();
    dir.close();
  }
  
  src.close();
}