org.apache.lucene.facet.taxonomy.TaxonomyReader Java Examples

The following examples show how to use org.apache.lucene.facet.taxonomy.TaxonomyReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** User drills down on 'Publish Date/2010', and we
 *  return facets for both 'Publish Date' and 'Author',
 *  using DrillSideways. */
private List<FacetResult> drillSideways() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Passing no baseQuery means we drill down on all
  // documents ("browse only"):
  DrillDownQuery q = new DrillDownQuery(config);

  // Now user drills down on Publish Date/2010:
  q.add("Publish Date", "2010");

  DrillSideways ds = new DrillSideways(searcher, config, taxoReader);
  DrillSidewaysResult result = ds.search(q, 10);

  // Retrieve results
  List<FacetResult> facets = result.facets.getAllDims(10);

  indexReader.close();
  taxoReader.close();
  
  return facets;
}
 
Example #2
Source File: DirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Add a new category into the index (and the cache), and return its new
 * ordinal.
 * <p>
 * Actually, we might also need to add some of the category's ancestors
 * before we can add the category itself (while keeping the invariant that a
 * parent is always added to the taxonomy before its child). We do this by
 * recursion.
 */
private int internalAddCategory(FacetLabel cp) throws IOException {
  // Find our parent's ordinal (recursively adding the parent category
  // to the taxonomy if it's not already there). Then add the parent
  // ordinal as payloads (rather than a stored field; payloads can be
  // more efficiently read into memory in bulk by LuceneTaxonomyReader)
  int parent;
  if (cp.length > 1) {
    FacetLabel parentPath = cp.subpath(cp.length - 1);
    parent = findCategory(parentPath);
    if (parent < 0) {
      parent = internalAddCategory(parentPath);
    }
  } else if (cp.length == 1) {
    parent = TaxonomyReader.ROOT_ORDINAL;
  } else {
    parent = TaxonomyReader.INVALID_ORDINAL;
  }
  int id = addCategoryDocument(cp, parent);

  return id;
}
 
Example #3
Source File: TaxonomyIndexArrays.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void computeChildrenSiblings(int first) {
  // reset the youngest child of all ordinals. while this should be done only
  // for the leaves, we don't know up front which are the leaves, so we reset
  // all of them.
  for (int i = first; i < parents.length; i++) {
    children[i] = TaxonomyReader.INVALID_ORDINAL;
  }
  
  // the root category has no parent, and therefore no siblings
  if (first == 0) {
    first = 1;
    siblings[0] = TaxonomyReader.INVALID_ORDINAL;
  }
  
  for (int i = first; i < parents.length; i++) {
    // note that parents[i] is always < i, so the right-hand-side of
    // the following line is already set when we get here
    siblings[i] = children[parents[i]];
    children[parents[i]] = i;
  }
}
 
Example #4
Source File: TestDirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testBackwardsCompatibility() throws Exception {
  // tests that if the taxonomy index doesn't have the INDEX_EPOCH
  // property (supports pre-3.6 indexes), all still works.
  Directory dir = newDirectory();
  
  // create an empty index first, so that DirTaxoWriter initializes indexEpoch to 1.
  new IndexWriter(dir, new IndexWriterConfig(null)).close();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
  taxoWriter.close();
  
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
  assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
  assertNull(TaxonomyReader.openIfChanged(taxoReader));
  taxoReader.close();
  
  dir.close();
}
 
Example #5
Source File: TestParallelDrillSideways.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected DrillSideways getNewDrillSidewaysBuildFacetsResult(IndexSearcher searcher, FacetsConfig config,
        TaxonomyReader taxoReader) {
  return new DrillSideways(searcher, config, taxoReader, null, executor) {
    @Override
    protected Facets buildFacetsResult(FacetsCollector drillDowns, FacetsCollector[] drillSideways,
            String[] drillSidewaysDims) throws IOException {
      Map<String, Facets> drillSidewaysFacets = new HashMap<>();
      Facets drillDownFacets = getTaxonomyFacetCounts(taxoReader, config, drillDowns);
      if (drillSideways != null) {
        for (int i = 0; i < drillSideways.length; i++) {
          drillSidewaysFacets.put(drillSidewaysDims[i], getTaxonomyFacetCounts(taxoReader, config, drillSideways[i]));
        }
      }

      if (drillSidewaysFacets.isEmpty()) {
        return drillDownFacets;
      } else {
        return new MultiFacets(drillSidewaysFacets, drillDownFacets);
      }

    }
  };
}
 
Example #6
Source File: TestDirectoryTaxonomyReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenIfChangedAndRefCount() throws Exception {
  Directory dir = new ByteBuffersDirectory(); // no need for random directories here

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.commit();

  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoReader.incRef();
  assertEquals("wrong refCount", 2, taxoReader.getRefCount());

  taxoWriter.addCategory(new FacetLabel("a", "b"));
  taxoWriter.commit();
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
  assertNotNull(newtr);
  taxoReader.close();
  taxoReader = newtr;
  assertEquals("wrong refCount", 1, taxoReader.getRefCount());

  taxoWriter.close();
  taxoReader.close();
  dir.close();
}
 
Example #7
Source File: TestDrillSideways.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected DrillSideways getNewDrillSidewaysBuildFacetsResult(IndexSearcher searcher, FacetsConfig config,
        TaxonomyReader taxoReader) {
  return new DrillSideways(searcher, config, taxoReader) {
    @Override
    protected Facets buildFacetsResult(FacetsCollector drillDowns, FacetsCollector[] drillSideways,
            String[] drillSidewaysDims) throws IOException {
      Map<String, Facets> drillSidewaysFacets = new HashMap<>();
      Facets drillDownFacets = getTaxonomyFacetCounts(taxoReader, config, drillDowns);
      if (drillSideways != null) {
        for (int i = 0; i < drillSideways.length; i++) {
          drillSidewaysFacets.put(drillSidewaysDims[i], getTaxonomyFacetCounts(taxoReader, config, drillSideways[i]));
        }
      }

      if (drillSidewaysFacets.isEmpty()) {
        return drillDownFacets;
      } else {
        return new MultiFacets(drillSidewaysFacets, drillDownFacets);
      }

    }
  };
}
 
Example #8
Source File: AssociationsFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** User drills down on 'tags/solr'. */
private FacetResult drillDown() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Passing no baseQuery means we drill down on all
  // documents ("browse only"):
  DrillDownQuery q = new DrillDownQuery(config);

  // Now user drills down on Publish Date/2010:
  q.add("tags", "solr");
  FacetsCollector fc = new FacetsCollector();
  FacetsCollector.search(searcher, q, 10, fc);

  // Retrieve results
  Facets facets = new TaxonomyFacetSumFloatAssociations("$genre", taxoReader, config, fc);
  FacetResult result = facets.getTopChildren(10, "genre");

  indexReader.close();
  taxoReader.close();
  
  return result;
}
 
Example #9
Source File: AssociationsFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** User runs a query and aggregates facets by summing their association values. */
private List<FacetResult> sumAssociations() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
  
  FacetsCollector fc = new FacetsCollector();
  
  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);
  
  Facets tags = new TaxonomyFacetSumIntAssociations("$tags", taxoReader, config, fc);
  Facets genre = new TaxonomyFacetSumFloatAssociations("$genre", taxoReader, config, fc);

  // Retrieve results
  List<FacetResult> results = new ArrayList<>();
  results.add(tags.getTopChildren(10, "tags"));
  results.add(genre.getTopChildren(10, "genre"));

  indexReader.close();
  taxoReader.close();
  
  return results;
}
 
Example #10
Source File: SimpleFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** User drills down on 'Publish Date/2010', and we
 *  return facets for 'Author' */
private FacetResult drillDown() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Passing no baseQuery means we drill down on all
  // documents ("browse only"):
  DrillDownQuery q = new DrillDownQuery(config);

  // Now user drills down on Publish Date/2010:
  q.add("Publish Date", "2010");
  FacetsCollector fc = new FacetsCollector();
  FacetsCollector.search(searcher, q, 10, fc);

  // Retrieve results
  Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
  FacetResult result = facets.getTopChildren(10, "Author");

  indexReader.close();
  taxoReader.close();
  
  return result;
}
 
Example #11
Source File: TestDrillSideways.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testEmptyIndex() throws Exception {
  // LUCENE-5045: make sure DrillSideways works with an empty index
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  IndexSearcher searcher = newSearcher(writer.getReader());
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  // Count "Author"
  FacetsConfig config = new FacetsConfig();
  DrillSideways ds = getNewDrillSideways(searcher, config, taxoReader);
  DrillDownQuery ddq = new DrillDownQuery(config);
  ddq.add("Author", "Lisa");

  DrillSidewaysResult r = ds.search(ddq, 10); // this used to fail on IllegalArgEx
  assertEquals(0, r.hits.totalHits.value);

  r = ds.search(ddq, null, null, 10, new Sort(new SortField("foo", SortField.Type.INT)), false); // this used to fail on IllegalArgEx
  assertEquals(0, r.hits.totalHits.value);

  writer.close();
  IOUtils.close(taxoWriter, searcher.getIndexReader(), taxoReader, dir, taxoDir);
}
 
Example #12
Source File: SimpleFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** User runs a query and counts facets. */
private List<FacetResult> facetsWithSearch() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  FacetsCollector fc = new FacetsCollector();

  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

  // Retrieve results
  List<FacetResult> results = new ArrayList<>();

  // Count both "Publish Date" and "Author" dimensions
  Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
  results.add(facets.getTopChildren(10, "Author"));
  results.add(facets.getTopChildren(10, "Publish Date"));
  
  indexReader.close();
  taxoReader.close();
  
  return results;
}
 
Example #13
Source File: LuceneNativeFacet.java    From orientdb-lucene with Apache License 2.0 6 votes vote down vote up
/** User runs a query and counts facets. */
private List<FacetResult> facetsWithSearch() throws IOException {
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

    FacetsCollector fc = new FacetsCollector();

    // MatchAllDocsQuery is for "browsing" (counts facets
    // for all non-deleted docs in the index); normally
    // you'd use a "normal" query:
    FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

    // Retrieve results
    List<FacetResult> results = new ArrayList<FacetResult>();

    // Count both "Publish Date" and "Author" dimensions
    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
    results.add(facets.getTopChildren(10, "Author"));
    results.add(facets.getTopChildren(10, "Publish Date"));

    indexReader.close();
    taxoReader.close();

    return results;
}
 
Example #14
Source File: LuceneNativeFacet.java    From orientdb-lucene with Apache License 2.0 6 votes vote down vote up
/** User drills down on 'Publish Date/2010', and we
 *  return facets for 'Author' */
private FacetResult drillDown() throws IOException {
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

    // Passing no baseQuery means we drill down on all
    // documents ("browse only"):
    DrillDownQuery q = new DrillDownQuery(config);

    // Now user drills down on Publish Date/2010:
    q.add("Publish Date", "2010");
    FacetsCollector fc = new FacetsCollector();
    FacetsCollector.search(searcher, q, 10, fc);

    // Retrieve results
    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
    FacetResult result = facets.getTopChildren(10, "Author");

    indexReader.close();
    taxoReader.close();

    return result;
}
 
Example #15
Source File: LuceneNativeFacet.java    From orientdb-lucene with Apache License 2.0 6 votes vote down vote up
/** User drills down on 'Publish Date/2010', and we
 *  return facets for both 'Publish Date' and 'Author',
 *  using DrillSideways. */
private List<FacetResult> drillSideways() throws IOException {
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

    // Passing no baseQuery means we drill down on all
    // documents ("browse only"):
    DrillDownQuery q = new DrillDownQuery(config);

    // Now user drills down on Publish Date/2010:
    q.add("Publish Date", "2010");

    DrillSideways ds = new DrillSideways(searcher, config, taxoReader);
    DrillSidewaysResult result = ds.search(q, 10);

    // Retrieve results

    List<FacetResult> facets = result.facets.getAllDims(10);

    indexReader.close();
    taxoReader.close();

    return facets;
}
 
Example #16
Source File: LumongoSegment.java    From lumongo with Apache License 2.0 6 votes vote down vote up
private void openReaderIfChanges() throws IOException {
	DirectoryReader newDirectoryReader = DirectoryReader
			.openIfChanged(directoryReader, indexWriter, indexConfig.getIndexSettings().getApplyUncommittedDeletes());
	if (newDirectoryReader != null) {
		directoryReader = newDirectoryReader;
		QueryResultCache qrc = queryResultCache;
		if (qrc != null) {
			qrc.clear();
		}

	}

	DirectoryTaxonomyReader newone = TaxonomyReader.openIfChanged(taxoReader);
	if (newone != null) {
		taxoReader = newone;
	}
}
 
Example #17
Source File: CloseTaxonomyReaderTask.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public int doLogic() throws IOException {
  try (TaxonomyReader taxoReader = getRunData().getTaxonomyReader()) {
    getRunData().setTaxonomyReader(null);
    if (taxoReader.getRefCount() != 1) {
      System.out.println("WARNING: CloseTaxonomyReader: reference count is currently " + taxoReader.getRefCount());
    }
  }
  return 1;
}
 
Example #18
Source File: TestParallelDrillSideways.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected DrillSideways getNewDrillSidewaysScoreSubdocsAtOnce(IndexSearcher searcher, FacetsConfig config,
        TaxonomyReader taxoReader) {
  return new DrillSideways(searcher, config, taxoReader, null, executor) {
    @Override
    protected boolean scoreSubDocsAtOnce() {
      return true;
    }
  };
}
 
Example #19
Source File: TestMultipleIndexFields.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testDefault() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
  FacetsConfig config = getConfig();

  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  // Obtain facets results and hand-test them
  assertCorrectResults(getTaxonomyFacetCounts(tr, config, sfc));

  assertOrdinalsExist("$facets", ir);

  iw.close();
  IOUtils.close(tr, ir, tw, indexDir, taxoDir);
}
 
Example #20
Source File: TestMultipleIndexFields.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testCustom() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Author", "$author");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  facetsMap.put("Author", getTaxonomyFacetCounts(tr, config, sfc, "$author"));
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);

  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$author", ir);

  iw.close();
  IOUtils.close(tr, ir, tw, indexDir, taxoDir);
}
 
Example #21
Source File: TestDirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testRecreateAndRefresh() throws Exception {
  // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
  // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
  // DirTaxoReader succeeding to refresh().
  try (Directory dir = newDirectory()) {

    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("a"));

    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);

    touchTaxo(taxoWriter, new FacetLabel("b"));

    TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));

    // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("c"));
    taxoWriter.close();

    taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    touchTaxo(taxoWriter, new FacetLabel("d"));
    taxoWriter.close();

    newtr = TaxonomyReader.openIfChanged(taxoReader);
    taxoReader.close();
    taxoReader = newtr;
    assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
    taxoReader.close();
  }
}
 
Example #22
Source File: TestDirectoryTaxonomyReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenIfChangedResult() throws Exception {
  Directory dir = null;
  DirectoryTaxonomyWriter ltw = null;
  DirectoryTaxonomyReader ltr = null;
  
  try {
    dir = newDirectory();
    ltw = new DirectoryTaxonomyWriter(dir);
    
    ltw.addCategory(new FacetLabel("a"));
    ltw.commit();
    
    ltr = new DirectoryTaxonomyReader(dir);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr));
    
    ltw.addCategory(new FacetLabel("b"));
    ltw.commit();
    
    DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr);
    assertNotNull("changes were committed", newtr);
    assertNull("Nothing has changed", TaxonomyReader.openIfChanged(newtr));
    newtr.close();
  } finally {
    IOUtils.close(ltw, ltr, dir);
  }
}
 
Example #23
Source File: TestDirectoryTaxonomyReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenIfChangedReuseAfterRecreate() throws Exception {
  // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
  FacetLabel cp_a = new FacetLabel("a");
  writer.addCategory(cp_a);
  writer.close();
  
  DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
  // fill r1's caches
  assertEquals(1, r1.getOrdinal(cp_a));
  assertEquals(cp_a, r1.getPath(1));
  
  // now recreate, add a different category
  writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
  FacetLabel cp_b = new FacetLabel("b");
  writer.addCategory(cp_b);
  writer.close();
  
  DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
  assertNotNull(r2);
  
  // fill r2's caches
  assertEquals(1, r2.getOrdinal(cp_b));
  assertEquals(cp_b, r2.getPath(1));
  
  // check that r1 doesn't see cp_b
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
  assertEquals(cp_a, r1.getPath(1));

  // check that r2 doesn't see cp_a
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
  assertEquals(cp_b, r2.getPath(1));

  r2.close();
  r1.close();
  dir.close();
}
 
Example #24
Source File: TestDirectoryTaxonomyReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenIfChangedReuse() throws Exception {
  // test the reuse of data from the old DTR instance
  for (boolean nrt : new boolean[] {false, true}) {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));
    
    FacetLabel cp_b = new FacetLabel("b");
    writer.addCategory(cp_b);
    if (!nrt) writer.commit();
    
    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);
    
    // add r2's categories to the caches
    assertEquals(2, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(2));
    
    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertNull(r1.getPath(2));
    
    r1.close();
    r2.close();
    writer.close();
    dir.close();
  }
}
 
Example #25
Source File: LuceneNativeFacet.java    From orientdb-lucene with Apache License 2.0 5 votes vote down vote up
/** User runs a query and counts facets only without collecting the matching documents.*/
private List<FacetResult> facetsOnly() throws IOException {
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

    FacetsCollector fc = new FacetsCollector();

    // MatchAllDocsQuery is for "browsing" (counts facets
    // for all non-deleted docs in the index); normally
    // you'd use a "normal" query:
    searcher.search(new MatchAllDocsQuery(), null /*Filter */, fc);

    // Retrieve results
    List<FacetResult> results = new ArrayList<FacetResult>();

    // Count both "Publish Date" and "Author" dimensions
    Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);

    results.add(facets.getTopChildren(10, "Author"));
    results.add(facets.getTopChildren(10, "Publish Date"));

    indexReader.close();
    taxoReader.close();

    return results;
}
 
Example #26
Source File: TestDrillSideways.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testScorer() throws Exception {
  // LUCENE-6001 some scorers, eg ReqExlScorer, can hit NPE if cost is called after nextDoc
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();

  // Writes facet ords to a separate directory from the
  // main index:
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);

  FacetsConfig config = new FacetsConfig();

  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(newTextField("field", "foo bar", Field.Store.NO));
  doc.add(new FacetField("Author", "Bob"));
  doc.add(new FacetField("dim", "a"));
  writer.addDocument(config.build(taxoWriter, doc));

  // NRT open
  IndexSearcher searcher = newSearcher(writer.getReader());

  // NRT open
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  DrillSideways ds = getNewDrillSideways(searcher, config, taxoReader);

  BooleanQuery.Builder bq = new BooleanQuery.Builder();
  bq.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST);
  bq.add(new TermQuery(new Term("field", "bar")), BooleanClause.Occur.MUST_NOT);
  DrillDownQuery ddq = new DrillDownQuery(config, bq.build());
  ddq.add("field", "foo");
  ddq.add("author", bq.build());
  ddq.add("dim", bq.build());
  DrillSidewaysResult r = ds.search(null, ddq, 10);
  assertEquals(0, r.hits.totalHits.value);

  writer.close();
  IOUtils.close(searcher.getIndexReader(), taxoReader, taxoWriter, dir, taxoDir);
}
 
Example #27
Source File: PerfRunData.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Set the taxonomy reader. Takes ownership of that taxonomy reader, that is,
 * internally performs taxoReader.incRef() (If caller no longer needs that 
 * reader it should decRef()/close() it after calling this method, otherwise, 
 * the reader will remain open). 
 * @param taxoReader The taxonomy reader to set.
 */
public synchronized void setTaxonomyReader(TaxonomyReader taxoReader) throws IOException {
  if (taxoReader == this.taxonomyReader) {
    return;
  }
  if (taxonomyReader != null) {
    taxonomyReader.decRef();
  }
  
  if (taxoReader != null) {
    taxoReader.incRef();
  }
  this.taxonomyReader = taxoReader;
}
 
Example #28
Source File: PerfRunData.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * @return Returns the taxonomyReader.  NOTE: this returns a
 * reference.  You must call TaxonomyReader.decRef() when
 * you're done.
 */
public synchronized TaxonomyReader getTaxonomyReader() {
  if (taxonomyReader != null) {
    taxonomyReader.incRef();
  }
  return taxonomyReader;
}
 
Example #29
Source File: TestPerfTasksLogic.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Test indexing with facets tasks.
 */
public void testIndexingWithFacets() throws Exception {
  // 1. alg definition (required in every "logic" test)
  String algLines[] = {
      "# ----- properties ",
      "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
      "docs.file=" + getReuters20LinesFile(),
      "content.source.log.step=100",
      "content.source.forever=false",
      "directory=ByteBuffersDirectory",
      "doc.stored=false",
      "merge.factor=3",
      "doc.tokenized=false",
      "debug.level=1",
      "# ----- alg ",
      "ResetSystemErase",
      "CreateIndex",
      "CreateTaxonomyIndex",
      "{ \"AddDocs\"  AddFacetedDoc > : * ",
      "CloseIndex",
      "CloseTaxonomyIndex",
      "OpenTaxonomyReader",
  };

  // 2. execute the algorithm  (required in every "logic" test)
  Benchmark benchmark = execBenchmark(algLines);
  PerfRunData runData = benchmark.getRunData();
  assertNull("taxo writer was not properly closed",runData.getTaxonomyWriter());
  TaxonomyReader taxoReader = runData.getTaxonomyReader();
  assertNotNull("taxo reader was not opened", taxoReader);
  assertTrue("nothing was added to the taxnomy (expecting root and at least one addtional category)",taxoReader.getSize()>1);
  taxoReader.close();
}
 
Example #30
Source File: ExpressionAggregationFacetsExample.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** User runs a query and aggregates facets. */
private FacetResult search() throws IOException, ParseException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Aggregate categories by an expression that combines the document's score
  // and its popularity field
  Expression expr = JavascriptCompiler.compile("_score * sqrt(popularity)");
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES); // the score of the document
  bindings.add("popularity", DoubleValuesSource.fromLongField("popularity")); // the value of the 'popularity' field

  // Aggregates the facet values
  FacetsCollector fc = new FacetsCollector(true);

  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

  // Retrieve results
  Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, expr.getDoubleValuesSource(bindings));
  FacetResult result = facets.getTopChildren(10, "A");
  
  indexReader.close();
  taxoReader.close();
  
  return result;
}