Java Code Examples for org.apache.lucene.facet.FacetsConfig

The following examples show how to use org.apache.lucene.facet.FacetsConfig. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: linden   Source File: ShardWriter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Process an intermediate form by carrying out, on the Lucene instance of
 * the shard, the deletes and the inserts (a ram index) in the form.
 * @param form  the intermediate form containing deletes and a ram index
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (facetsConfig != null) {
    DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
    // merge the taxonomies
    taxoWriter.addTaxonomy(form.getTaxoDirectory(), map);
    int ordinalMap[] = map.getMap();
    DirectoryReader reader = DirectoryReader.open(form.getDirectory());
    try {
      List<AtomicReaderContext> leaves = reader.leaves();
      int numReaders = leaves.size();
      AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
      for (int i = 0; i < numReaders; i++) {
        wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
      }
      writer.addIndexes(new MultiReader(wrappedLeaves));
    } finally {
      reader.close();
    }
  } else {
    writer.addIndexes(new Directory[] { form.getDirectory() });
  }
  numForms++;
}
 
Example 2
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  publishIndexDir = newDirectory();
  publishTaxoDir = newDirectory();
  handlerIndexDir = newMockDirectory();
  handlerTaxoDir = newMockDirectory();
  clientWorkDir = createTempDir("replicationClientTest");
  sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
  replicator = new LocalReplicator();
  callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
  handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
  client = new ReplicationClient(replicator, handler, sourceDirFactory);
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  publishIndexWriter = new IndexWriter(publishIndexDir, conf);
  publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
  config = new FacetsConfig();
  config.setHierarchical("A", true);
}
 
Example 3
Source Project: lucene-solr   Source File: TaxonomyMergeUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Merges the given taxonomy and index directories and commits the changes to
 * the given writers.
 */
public static void merge(Directory srcIndexDir, Directory srcTaxoDir, OrdinalMap map, IndexWriter destIndexWriter,
    DirectoryTaxonomyWriter destTaxoWriter, FacetsConfig srcConfig) throws IOException {
  
  // merge the taxonomies
  destTaxoWriter.addTaxonomy(srcTaxoDir, map);
  int ordinalMap[] = map.getMap();
  DirectoryReader reader = DirectoryReader.open(srcIndexDir);
  try {
    List<LeafReaderContext> leaves = reader.leaves();
    int numReaders = leaves.size();
    CodecReader wrappedLeaves[] = new CodecReader[numReaders];
    for (int i = 0; i < numReaders; i++) {
      wrappedLeaves[i] = SlowCodecReaderWrapper.wrap(new OrdinalMappingLeafReader(leaves.get(i).reader(), ordinalMap, srcConfig));
    }
    destIndexWriter.addIndexes(wrappedLeaves);
    
    // commit changes to taxonomy and index respectively.
    destTaxoWriter.commit();
    destIndexWriter.commit();
  } finally {
    reader.close();
  }
}
 
Example 4
Source Project: lucene-solr   Source File: TaxonomyFacets.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<FacetResult> getAllDims(int topN) throws IOException {
  int[] children = getChildren();
  int[] siblings = getSiblings();
  int ord = children[TaxonomyReader.ROOT_ORDINAL];
  List<FacetResult> results = new ArrayList<>();
  while (ord != TaxonomyReader.INVALID_ORDINAL) {
    String dim = taxoReader.getPath(ord).components[0];
    FacetsConfig.DimConfig dimConfig = config.getDimConfig(dim);
    if (dimConfig.indexFieldName.equals(indexFieldName)) {
      FacetResult result = getTopChildren(topN, dim);
      if (result != null) {
        results.add(result);
      }
    }
    ord = siblings[ord];
  }

  // Sort by highest value, tie break by dim:
  Collections.sort(results, BY_VALUE_THEN_DIM);
  return results;
}
 
Example 5
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesInSameIndexField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  doc.add(new FloatAssociationFacetField(55.0f, "b", "y"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });
  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 6
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoHierarchy() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  config.setHierarchical("a", true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 7
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testRequireDimCount() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  config.setRequireDimCount("a", true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 8
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testReallyNoNormsForDrillDown() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setSimilarity(new PerFieldSimilarityWrapper() {
      final Similarity sim = new ClassicSimilarity();

      @Override
      public Similarity get(String name) {
        assertEquals("field", name);
        return sim;
      }
    });
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  writer.addDocument(config.build(taxoWriter, doc));
  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 9
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testDetectHierarchicalField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path", "other"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 10
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testDetectMultiValuedField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  doc.add(new FacetField("a", "path2"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 11
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testGetFacetResultsTwice() throws Exception {
  // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(new FacetField("a", "1"));
  doc.add(new FacetField("b", "1"));
  iw.addDocument(config.build(taxoWriter, doc));
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
  
  List<FacetResult> res1 = facets.getAllDims(10);
  List<FacetResult> res2 = facets.getAllDims(10);
  assertEquals("calling getFacetResults twice should return the .equals()=true result", res1, res2);

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 12
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testChildCount() throws Exception {
  // LUCENE-4885: FacetResult.numValidDescendants was not set properly by FacetsAccumulator
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();
  for (int i = 0; i < 10; i++) {
    Document doc = new Document();
    doc.add(new FacetField("a", Integer.toString(i)));
    iw.addDocument(config.build(taxoWriter, doc));
  }
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
  
  Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
  
  assertEquals(10, facets.getTopChildren(2, "a").childCount);

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 13
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
private void indexTwoDocs(TaxonomyWriter taxoWriter, IndexWriter indexWriter, FacetsConfig config, boolean withContent) throws Exception {
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    if (withContent) {
      doc.add(new StringField("f", "a", Field.Store.NO));
    }
    if (config != null) {
      doc.add(new FacetField("A", Integer.toString(i)));
      indexWriter.addDocument(config.build(taxoWriter, doc));
    } else {
      indexWriter.addDocument(doc);
    }
  }
  
  indexWriter.commit();
}
 
Example 14
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
private static Facets getAllFacets(String indexFieldName, IndexSearcher searcher, TaxonomyReader taxoReader, FacetsConfig config) throws IOException {
  if (random().nextBoolean()) {
    // Aggregate the facet counts:
    FacetsCollector c = new FacetsCollector();

    // MatchAllDocsQuery is for "browsing" (counts facets
    // for all non-deleted docs in the index); normally
    // you'd use a "normal" query, and use MultiCollector to
    // wrap collecting the "normal" hits and also facets:
    searcher.search(new MatchAllDocsQuery(), c);

    return new FastTaxonomyFacetCounts(taxoReader, config, c);
  } else {
    return new FastTaxonomyFacetCounts(indexFieldName, searcher.getIndexReader(), taxoReader, config);
  }
}
 
Example 15
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts2.java    License: Apache License 2.0 6 votes vote down vote up
private static void addFacets(Document doc, FacetsConfig config, boolean updateTermExpectedCounts) 
    throws IOException {
  List<FacetField> docCategories = randomCategories(random());
  for (FacetField ff : docCategories) {
    doc.add(ff);
    String cp = ff.dim + "/" + ff.path[0];
    allExpectedCounts.put(cp, allExpectedCounts.get(cp) + 1);
    if (updateTermExpectedCounts) {
      termExpectedCounts.put(cp, termExpectedCounts.get(cp) + 1);
    }
  }
  // add 1 to each NO_PARENTS dimension
  allExpectedCounts.put(CP_B, allExpectedCounts.get(CP_B) + 1);
  allExpectedCounts.put(CP_C, allExpectedCounts.get(CP_C) + 1);
  allExpectedCounts.put(CP_D, allExpectedCounts.get(CP_D) + 1);
  if (updateTermExpectedCounts) {
    termExpectedCounts.put(CP_B, termExpectedCounts.get(CP_B) + 1);
    termExpectedCounts.put(CP_C, termExpectedCounts.get(CP_C) + 1);
    termExpectedCounts.put(CP_D, termExpectedCounts.get(CP_D) + 1);
  }
}
 
Example 16
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts2.java    License: Apache License 2.0 6 votes vote down vote up
private static void indexDocsWithFacetsAndSomeTerms(IndexWriter indexWriter, TaxonomyWriter taxoWriter, 
                                                    Map<String,Integer> expectedCounts) throws IOException {
  Random random = random();
  int numDocs = atLeast(random, 2);
  FacetsConfig config = getConfig();
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    boolean hasContent = random.nextBoolean();
    if (hasContent) {
      addField(doc);
    }
    addFacets(doc, config, hasContent);
    indexWriter.addDocument(config.build(taxoWriter, doc));
  }
  indexWriter.commit(); // flush a segment
}
 
Example 17
public void testNoScore() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();
  for (int i = 0; i < 4; i++) {
    Document doc = new Document();
    doc.add(new NumericDocValuesField("price", (i+1)));
    doc.add(new FacetField("a", Integer.toString(i % 2)));
    iw.addDocument(config.build(taxoWriter, doc));
  }
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  FacetsCollector sfc = new FacetsCollector();
  newSearcher(r).search(new MatchAllDocsQuery(), sfc);
  Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, DoubleValuesSource.fromLongField("price"));
  assertEquals("dim=a path=[] value=10.0 childCount=2\n  1 (6.0)\n  0 (4.0)\n", facets.getTopChildren(10, "a").toString());

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 18
Source Project: linden   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method is used by the index update mapper and process a document
 * operation into the current intermediate form.
 * @param doc  input document operation
 * @throws IOException
 */
public void process(Term id, Document doc, Analyzer analyzer, FacetsConfig facetsConfig) throws IOException {
  if (writer == null) {
    // analyzer is null because we specify an analyzer with addDocument
    createWriter();
  }
  if (facetsConfig != null) {
    writer.updateDocument(id, facetsConfig.build(taxoWriter, doc), analyzer);
  } else {
    writer.updateDocument(id, doc, analyzer);
  }
  numDocs++;
}
 
Example 19
Source Project: linden   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method is used by the index update combiner and process an
 * intermediate form into the current intermediate form. More specifically,
 * the input intermediate forms are a single-document ram index and/or a
 * single delete term.
 * @param form  the input intermediate form
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (form.dir.ramBytesUsed() > 0 || form.taxoDir.ramBytesUsed() > 0) {
    if (writer == null) {
      createWriter();
    }

    if (facetsConfig != null) {
      DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
      // merge the taxonomies
      taxoWriter.addTaxonomy(form.taxoDir, map);
      int ordinalMap[] = map.getMap();
      DirectoryReader reader = DirectoryReader.open(form.dir);
      try {
        List<AtomicReaderContext> leaves = reader.leaves();
        int numReaders = leaves.size();
        AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
        for (int i = 0; i < numReaders; i++) {
          wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
        }
        writer.addIndexes(new MultiReader(wrappedLeaves));
      } finally {
        reader.close();
      }
    } else {
      writer.addIndexes(new Directory[] { form.dir });
    }
    numDocs++;
  }
}
 
Example 20
Source Project: lucene-solr   Source File: RandomFacetSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void configure(FacetsConfig config) {
  for(int i=0;i<maxDims;i++) {
    config.setHierarchical(Integer.toString(i), true);
    config.setMultiValued(Integer.toString(i), true);
  }
}
 
Example 21
Source Project: lucene-solr   Source File: AssociationsFacetsExample.java    License: Apache License 2.0 5 votes vote down vote up
/** Empty constructor */
public AssociationsFacetsExample() {
  config = new FacetsConfig();
  config.setMultiValued("tags", true);
  config.setIndexFieldName("tags", "$tags");
  config.setMultiValued("genre", true);
  config.setIndexFieldName("genre", "$genre");
}
 
Example 22
public IndexAndTaxonomyReadyCallback(Directory indexDir, Directory taxoDir) throws IOException {
  this.indexDir = indexDir;
  this.taxoDir = taxoDir;
  config = new FacetsConfig();
  config.setHierarchical("A", true);
  if (DirectoryReader.indexExists(indexDir)) {
    indexReader = DirectoryReader.open(indexDir);
    lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
    taxoReader = new DirectoryTaxonomyReader(taxoDir);
  }
}
 
Example 23
Source Project: lucene-solr   Source File: SortedSetDocValuesFacetCounts.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Number getSpecificValue(String dim, String... path) throws IOException {
  if (path.length != 1) {
    throw new IllegalArgumentException("path must be length=1");
  }
  int ord = (int) dv.lookupTerm(new BytesRef(FacetsConfig.pathToString(dim, path)));
  if (ord < 0) {
    return -1;
  }

  return counts[ord];
}
 
Example 24
@Override
public Number getSpecificValue(String dim, String... path) throws IOException {
  if (path.length != 1) {
    throw new IllegalArgumentException("path must be length=1");
  }
  int ord = (int) dv.lookupTerm(new BytesRef(FacetsConfig.pathToString(dim, path)));
  if (ord < 0) {
    return -1;
  }

  return counts.get(ord);
}
 
Example 25
Source Project: lucene-solr   Source File: TaxonomyFacetSumValueSource.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Aggreggates float facet values from the provided
 *  {@link DoubleValuesSource}, and pulls ordinals from the
 *  provided {@link OrdinalsReader}.
 */
public TaxonomyFacetSumValueSource(OrdinalsReader ordinalsReader, TaxonomyReader taxoReader,
                                   FacetsConfig config, FacetsCollector fc, DoubleValuesSource vs) throws IOException {
  super(ordinalsReader.getIndexFieldName(), taxoReader, config);
  this.ordinalsReader = ordinalsReader;
  sumValues(fc.getMatchingDocs(), fc.getKeepScores(), vs);
}
 
Example 26
Source Project: lucene-solr   Source File: IntTaxonomyFacets.java    License: Apache License 2.0 5 votes vote down vote up
/** Sole constructor. */
protected IntTaxonomyFacets(String indexFieldName, TaxonomyReader taxoReader, FacetsConfig config, FacetsCollector fc) throws IOException {
  super(indexFieldName, taxoReader, config);

  if (useHashTable(fc, taxoReader)) {
    sparseValues = new IntIntScatterMap();
    values = null;
  } else {
    sparseValues = null;
    values = new int[taxoReader.getSize()];
  }
}
 
Example 27
Source Project: lucene-solr   Source File: DirectoryTaxonomyWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Note that the methods calling addCategoryDocument() are synchornized, so
 * this method is effectively synchronized as well.
 */
private int addCategoryDocument(FacetLabel categoryPath, int parent) throws IOException {
  // Before Lucene 2.9, position increments >=0 were supported, so we
  // added 1 to parent to allow the parent -1 (the parent of the root).
  // Unfortunately, starting with Lucene 2.9, after LUCENE-1542, this is
  // no longer enough, since 0 is not encoded consistently either (see
  // comment in SinglePositionTokenStream). But because we must be
  // backward-compatible with existing indexes, we can't just fix what
  // we write here (e.g., to write parent+2), and need to do a workaround
  // in the reader (which knows that anyway only category 0 has a parent
  // -1).    
  parentStream.set(Math.max(parent + 1, 1));
  Document d = new Document();
  d.add(parentStreamField);

  fullPathField.setStringValue(FacetsConfig.pathToString(categoryPath.components, categoryPath.length));
  d.add(fullPathField);

  // Note that we do no pass an Analyzer here because the fields that are
  // added to the Document are untokenized or contains their own TokenStream.
  // Therefore the IndexWriter's Analyzer has no effect.
  indexWriter.addDocument(d);
  int id = nextID++;

  // added a category document, mark that ReaderManager is not up-to-date
  shouldRefreshReaderManager = true;
  
  // also add to the parent array
  taxoArrays = getTaxoArrays().add(id, parent);

  // NOTE: this line must be executed last, or else the cache gets updated
  // before the parents array (LUCENE-4596)
  addToCache(categoryPath, id);

  return id;
}
 
Example 28
Source Project: lucene-solr   Source File: DirectoryTaxonomyWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Takes the categories from the given taxonomy directory, and adds the
 * missing ones to this taxonomy. Additionally, it fills the given
 * {@link OrdinalMap} with a mapping from the original ordinal to the new
 * ordinal.
 */
public void addTaxonomy(Directory taxoDir, OrdinalMap map) throws IOException {
  ensureOpen();
  DirectoryReader r = DirectoryReader.open(taxoDir);
  try {
    final int size = r.numDocs();
    final OrdinalMap ordinalMap = map;
    ordinalMap.setSize(size);
    int base = 0;
    PostingsEnum docs = null;
    for (final LeafReaderContext ctx : r.leaves()) {
      final LeafReader ar = ctx.reader();
      final Terms terms = ar.terms(Consts.FULL);
      // TODO: share per-segment TermsEnum here!
      TermsEnum te = terms.iterator();
      while (te.next() != null) {
        FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(te.term().utf8ToString()));
        final int ordinal = addCategory(cp);
        docs = te.postings(docs, PostingsEnum.NONE);
        ordinalMap.addMapping(docs.nextDoc() + base, ordinal);
      }
      base += ar.maxDoc(); // no deletions, so we're ok
    }
    ordinalMap.addDone();
  } finally {
    r.close();
  }
}
 
Example 29
Source Project: lucene-solr   Source File: DirectoryTaxonomyReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FacetLabel getPath(int ordinal) throws IOException {
  ensureOpen();
  
  // Since the cache is shared with DTR instances allocated from
  // doOpenIfChanged, we need to ensure that the ordinal is one that this DTR
  // instance recognizes. Therefore we do this check up front, before we hit
  // the cache.
  if (ordinal < 0 || ordinal >= indexReader.maxDoc()) {
    return null;
  }
  
  // TODO: can we use an int-based hash impl, such as IntToObjectMap,
  // wrapped as LRU?
  Integer catIDInteger = Integer.valueOf(ordinal);
  synchronized (categoryCache) {
    FacetLabel res = categoryCache.get(catIDInteger);
    if (res != null) {
      return res;
    }
  }
  
  Document doc = indexReader.document(ordinal);
  FacetLabel ret = new FacetLabel(FacetsConfig.stringToPath(doc.get(Consts.FULL)));
  synchronized (categoryCache) {
    categoryCache.put(catIDInteger, ret);
  }
  
  return ret;
}
 
Example 30
Source Project: lucene-solr   Source File: OrdinalMappingLeafReader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Wraps an LeafReader, mapping ordinals according to the ordinalMap, using
 * the provided {@link FacetsConfig} which was used to build the wrapped
 * reader.
 */
public OrdinalMappingLeafReader(LeafReader in, int[] ordinalMap, FacetsConfig srcConfig) {
  super(in);
  this.ordinalMap = ordinalMap;
  facetsConfig = new InnerFacetsConfig();
  facetFields = new HashSet<>();
  for (DimConfig dc : srcConfig.getDimConfigs().values()) {
    facetFields.add(dc.indexFieldName);
  }
  // always add the default indexFieldName. This is because FacetsConfig does
  // not explicitly record dimensions that were indexed under the default
  // DimConfig, unless they have a custome DimConfig.
  facetFields.add(FacetsConfig.DEFAULT_DIM_CONFIG.indexFieldName);
}