Java Code Examples for org.apache.lucene.facet.taxonomy.TaxonomyReader#openIfChanged()

The following examples show how to use org.apache.lucene.facet.taxonomy.TaxonomyReader#openIfChanged() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testOpenIfChangedAndRefCount() throws Exception {
  Directory dir = new ByteBuffersDirectory(); // no need for random directories here

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.commit();

  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoReader.incRef();
  assertEquals("wrong refCount", 2, taxoReader.getRefCount());

  taxoWriter.addCategory(new FacetLabel("a", "b"));
  taxoWriter.commit();
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
  assertNotNull(newtr);
  taxoReader.close();
  taxoReader = newtr;
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoWriter.close();
  taxoReader.close();
  dir.close();
}
 
Example 2
Source Project: lumongo   File: LumongoSegment.java    License: Apache License 2.0 6 votes vote down vote up
private void openReaderIfChanges() throws IOException {
	DirectoryReader newDirectoryReader = DirectoryReader
			.openIfChanged(directoryReader, indexWriter, indexConfig.getIndexSettings().getApplyUncommittedDeletes());
	if (newDirectoryReader != null) {
		directoryReader = newDirectoryReader;
		QueryResultCache qrc = queryResultCache;
		if (qrc != null) {
			qrc.clear();
		}

	}

	DirectoryTaxonomyReader newone = TaxonomyReader.openIfChanged(taxoReader);
	if (newone != null) {
		taxoReader = newone;
	}
}
 
Example 3
@Override
public Boolean call() throws Exception {
  if (indexReader == null) {
    indexReader = DirectoryReader.open(indexDir);
    lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
    taxoReader = new DirectoryTaxonomyReader(taxoDir);
  } else {
    // verify search index
    DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
    assertNotNull("should not have reached here if no changes were made to the index", newReader);
    long newGeneration = newReader.getIndexCommit().getGeneration();
    assertTrue("expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration, newGeneration > lastIndexGeneration);
    indexReader.close();
    indexReader = newReader;
    lastIndexGeneration = newGeneration;
    TestUtil.checkIndex(indexDir);
    
    // verify taxonomy index
    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
    if (newTaxoReader != null) {
      taxoReader.close();
      taxoReader = newTaxoReader;
    }
    TestUtil.checkIndex(taxoDir);
    
    // verify faceted search
    int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    FacetsCollector fc = new FacetsCollector();
    searcher.search(new MatchAllDocsQuery(), fc);
    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
    assertEquals(1, facets.getSpecificValue("A", Integer.toString(id, 16)).intValue());
    
    DrillDownQuery drillDown = new DrillDownQuery(config);
    drillDown.add("A", Integer.toString(id, 16));
    TopDocs docs = searcher.search(drillDown, 10);
    assertEquals(1, docs.totalHits.value);
  }
  return null;
}
 
Example 4
@Test
public void testRecreateAndRefresh() throws Exception {
  // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
  // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
  // DirTaxoReader succeeding to refresh().
  try (Directory dir = newDirectory()) {

    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("a"));

    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);

    touchTaxo(taxoWriter, new FacetLabel("b"));

    TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));

    // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("c"));
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("d"));
    taxoWriter.close();

    newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
    taxoReader.close();
  }
}
 
Example 5
@Test
public void testOpenIfChangedResult() throws Exception {
  Directory dir = null;
  DirectoryTaxonomyWriter ltw = null;
  DirectoryTaxonomyReader ltr = null;
  
  try {
    dir = newDirectory();
    ltw = new DirectoryTaxonomyWriter(dir);
    
    ltw.addCategory(new FacetLabel("a"));
    ltw.commit();
    
    ltr = new DirectoryTaxonomyReader(dir);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr));
    
    ltw.addCategory(new FacetLabel("b"));
    ltw.commit();
    
    DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr);
    assertNotNull("changes were committed", newtr);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(newtr));
    newtr.close();
  } finally {
    IOUtils.close(ltw, ltr, dir);
  }
}
 
Example 6
@Test
public void testOpenIfChangedReuseAfterRecreate() throws Exception {
  // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
  FacetLabel cp_a = new FacetLabel("a");
  writer.addCategory(cp_a);
  writer.close();
  
  DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
  // fill r1's caches
  assertEquals(1, r1.getOrdinal(cp_a));
  assertEquals(cp_a, r1.getPath(1));
  
  // now recreate, add a different category
  writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
  FacetLabel cp_b = new FacetLabel("b");
  writer.addCategory(cp_b);
  writer.close();
  
  DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
  assertNotNull(r2);
  
  // fill r2's caches
  assertEquals(1, r2.getOrdinal(cp_b));
  assertEquals(cp_b, r2.getPath(1));
  
  // check that r1 doesn't see cp_b
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
  assertEquals(cp_a, r1.getPath(1));

  // check that r2 doesn't see cp_a
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
  assertEquals(cp_b, r2.getPath(1));

  r2.close();
  r1.close();
  dir.close();
}
 
Example 7
@Test
public void testOpenIfChangedReuse() throws Exception {
  // test the reuse of data from the old DTR instance
  for (boolean nrt : new boolean[] {false, true}) {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));
    
    FacetLabel cp_b = new FacetLabel("b");
    writer.addCategory(cp_b);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);
    
    // add r2's categories to the caches
    assertEquals(2, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(2));
    
    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertNull(r1.getPath(2));
    
    r1.close();
    r2.close();
    writer.close();
    dir.close();
  }
}
 
Example 8
private void doTestReadRecreatedTaxonomy(Random random, boolean closeReader) throws Exception {
  Directory dir = null;
  TaxonomyWriter tw = null;
  TaxonomyReader tr = null;
  
  // prepare a few categories
  int  n = 10;
  FacetLabel[] cp = new FacetLabel[n];
  for (int i=0; i<n; i++) {
    cp[i] = new FacetLabel("a", Integer.toString(i));
  }
  
  try {
    dir = newDirectory();
    
    tw = new DirectoryTaxonomyWriter(dir);
    tw.addCategory(new FacetLabel("a"));
    tw.close();
    
    tr = new DirectoryTaxonomyReader(dir);
    int baseNumCategories = tr.getSize();
    
    for (int i=0; i<n; i++) {
      int k = random.nextInt(n);
      tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
      for (int j = 0; j <= k; j++) {
        tw.addCategory(cp[j]);
      }
      tw.close();
      if (closeReader) {
        tr.close();
        tr = new DirectoryTaxonomyReader(dir);
      } else {
        TaxonomyReader newtr = TaxonomyReader.openIfChanged(tr);
        assertNotNull(newtr);
        tr.close();
        tr = newtr;
      }
      assertEquals("Wrong #categories in taxonomy (i="+i+", k="+k+")", baseNumCategories + 1 + k, tr.getSize());
    }
  } finally {
    IOUtils.close(tr, tw, dir);
  }
}
 
Example 9
@Test
public void testOpenIfChangedManySegments() throws Exception {
  // test openIfChanged() when the taxonomy contains many segments
  Directory dir = newDirectory();
  
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
      IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
      lmp.setMergeFactor(2);
      return conf;
    }
  };
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  
  int numRounds = random().nextInt(10) + 10;
  int numCategories = 1; // one for root
  for (int i = 0; i < numRounds; i++) {
    int numCats = random().nextInt(4) + 1;
    for (int j = 0; j < numCats; j++) {
      writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j)));
    }
    numCategories += numCats + 1 /* one for round-parent */;
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
    assertNotNull(newtr);
    reader.close();
    reader = newtr;
    
    // assert categories
    assertEquals(numCategories, reader.getSize());
    int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i)));
    int[] parents = reader.getParallelTaxonomyArrays().parents();
    assertEquals(0, parents[roundOrdinal]); // round's parent is root
    for (int j = 0; j < numCats; j++) {
      int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j)));
      assertEquals(roundOrdinal, parents[ord]); // round's parent is root
    }
  }
  
  reader.close();
  writer.close();
  dir.close();
}
 
Example 10
@Test
public void testOpenIfChangedMergedSegment() throws Exception {
  // test openIfChanged() when all index segments were merged - used to be
  // a bug in ParentArray, caught by testOpenIfChangedManySegments - only
  // this test is not random
  Directory dir = newDirectory();
  
  // hold onto IW to forceMerge
  // note how we don't close it, since DTW will close it.
  final IndexWriter iw = new IndexWriter(dir,
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setMergePolicy(new LogByteSizeMergePolicy()));
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriter openIndexWriter(Directory directory,
        IndexWriterConfig config) throws IOException {
      return iw;
    }
  };
  
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  assertEquals(1, reader.getSize());
  assertEquals(1, reader.getParallelTaxonomyArrays().parents().length);

  // add category and call forceMerge -- this should flush IW and merge segments down to 1
  // in ParentArray.initFromReader, this used to fail assuming there are no parents.
  writer.addCategory(new FacetLabel("1"));
  iw.forceMerge(1);
  
  // now calling openIfChanged should trip on the bug
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
  assertNotNull(newtr);
  reader.close();
  reader = newtr;
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
  
  reader.close();
  writer.close();
  dir.close();
}
 
Example 11
@Test
public void testOpenIfChangedNoChangesButSegmentMerges() throws Exception {
  // test openIfChanged() when the taxonomy hasn't really changed, but segments
  // were merged. The NRT reader will be reopened, and ParentArray used to assert
  // that the new reader contains more ordinals than were given from the old
  // TaxReader version
  Directory dir = newDirectory();
  
  // hold onto IW to forceMerge
  // note how we don't close it, since DTW will close it.
  final IndexWriter iw = new IndexWriter(dir,
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setMergePolicy(new LogByteSizeMergePolicy()));
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriter openIndexWriter(Directory directory,
        IndexWriterConfig config) throws IOException {
      return iw;
    }
  };
  
  // add a category so that the following DTR open will cause a flush and 
  // a new segment will be created
  writer.addCategory(new FacetLabel("a"));
  
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);

  // merge all the segments so that NRT reader thinks there's a change 
  iw.forceMerge(1);
  
  // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
  assertNotNull(newtr);
  reader.close();
  reader = newtr;
  assertEquals(2, reader.getSize());
  assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
  
  reader.close();
  writer.close();
  dir.close();
}
 
Example 12
@Test
public void testOpenIfChangedReplaceTaxonomy() throws Exception {
  // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
  // only can work with NRT as well
  Directory src = newDirectory();
  DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
  FacetLabel cp_b = new FacetLabel("b");
  w.addCategory(cp_b);
  w.close();
  
  for (boolean nrt : new boolean[] {false, true}) {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));

    // now replace taxonomy
    writer.replaceTaxonomy(src);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);
    
    // fill r2's caches
    assertEquals(1, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(1));
    
    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertEquals(cp_a, r1.getPath(1));

    // check that r2 doesn't see cp_a
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
    assertEquals(cp_b, r2.getPath(1));

    r2.close();
    r1.close();
    writer.close();
    dir.close();
  }
  
  src.close();
}