Java Code Examples for org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter

The following examples show how to use org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: linden   Source File: ShardWriter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Process an intermediate form by carrying out, on the Lucene instance of
 * the shard, the deletes and the inserts (a ram index) in the form.
 * @param form  the intermediate form containing deletes and a ram index
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (facetsConfig != null) {
    DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
    // merge the taxonomies
    taxoWriter.addTaxonomy(form.getTaxoDirectory(), map);
    int ordinalMap[] = map.getMap();
    DirectoryReader reader = DirectoryReader.open(form.getDirectory());
    try {
      List<AtomicReaderContext> leaves = reader.leaves();
      int numReaders = leaves.size();
      AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
      for (int i = 0; i < numReaders; i++) {
        wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
      }
      writer.addIndexes(new MultiReader(wrappedLeaves));
    } finally {
      reader.close();
    }
  } else {
    writer.addIndexes(new Directory[] { form.getDirectory() });
  }
  numForms++;
}
 
Example 2
Source Project: linden   Source File: LindenCoreImpl.java    License: Apache License 2.0 6 votes vote down vote up
public LindenCoreImpl(LindenConfig lindenConfig, String subIndexDirectory) throws IOException {
  this.config = lindenConfig;
  idFieldName = config.getSchema().getId();
  facetsConfig = config.createFacetsConfig();

  String directory = config.getIndexDirectory();
  if (subIndexDirectory != null) {
    directory = FilenameUtils.concat(config.getIndexDirectory(), subIndexDirectory);
  }

  indexWriter =
      new IndexWriter(createIndexDirectory(directory, config.getIndexType()), config.createIndexWriterConfig());
  trackingIndexWriter = new TrackingIndexWriter(indexWriter);

  taxoWriter =
      facetsConfig != null ? new DirectoryTaxonomyWriter(createTaxoIndexDirectory(directory, config.getIndexType()))
                           : null;
  commitStrategy = new CommitStrategy(indexWriter, taxoWriter);
  commitStrategy.start();

  lindenNRTSearcherManager = new LindenNRTSearcherManager(config,
                                                          trackingIndexWriter, taxoWriter);
  snippetGenerator = new LindenSnippetGenerator();
}
 
Example 3
Source Project: lucene-solr   Source File: TaxonomyMergeUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Merges the given taxonomy and index directories and commits the changes to
 * the given writers.
 */
public static void merge(Directory srcIndexDir, Directory srcTaxoDir, OrdinalMap map, IndexWriter destIndexWriter,
    DirectoryTaxonomyWriter destTaxoWriter, FacetsConfig srcConfig) throws IOException {
  
  // merge the taxonomies
  destTaxoWriter.addTaxonomy(srcTaxoDir, map);
  int ordinalMap[] = map.getMap();
  DirectoryReader reader = DirectoryReader.open(srcIndexDir);
  try {
    List<LeafReaderContext> leaves = reader.leaves();
    int numReaders = leaves.size();
    CodecReader wrappedLeaves[] = new CodecReader[numReaders];
    for (int i = 0; i < numReaders; i++) {
      wrappedLeaves[i] = SlowCodecReaderWrapper.wrap(new OrdinalMappingLeafReader(leaves.get(i).reader(), ordinalMap, srcConfig));
    }
    destIndexWriter.addIndexes(wrappedLeaves);
    
    // commit changes to taxonomy and index respectively.
    destTaxoWriter.commit();
    destIndexWriter.commit();
  } finally {
    reader.close();
  }
}
 
Example 4
Source Project: lucene-solr   Source File: TestDrillSideways.java    License: Apache License 2.0 6 votes vote down vote up
public void testEmptyIndex() throws Exception {
  // LUCENE-5045: make sure DrillSideways works with an empty index
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  IndexSearcher searcher = newSearcher(writer.getReader());
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  // Count "Author"
  FacetsConfig config = new FacetsConfig();
  DrillSideways ds = getNewDrillSideways(searcher, config, taxoReader);
  DrillDownQuery ddq = new DrillDownQuery(config);
  ddq.add("Author", "Lisa");

  DrillSidewaysResult r = ds.search(ddq, 10); // this used to fail on IllegalArgEx
  assertEquals(0, r.hits.totalHits.value);

  r = ds.search(ddq, null, null, 10, new Sort(new SortField("foo", SortField.Type.INT)), false); // this used to fail on IllegalArgEx
  assertEquals(0, r.hits.totalHits.value);

  writer.close();
  IOUtils.close(taxoWriter, searcher.getIndexReader(), taxoReader, dir, taxoDir);
}
 
Example 5
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesInSameIndexField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  doc.add(new FloatAssociationFacetField(55.0f, "b", "y"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });
  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 6
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoHierarchy() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  config.setHierarchical("a", true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 7
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testRequireDimCount() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  config.setRequireDimCount("a", true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 8
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testReallyNoNormsForDrillDown() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setSimilarity(new PerFieldSimilarityWrapper() {
      final Similarity sim = new ClassicSimilarity();

      @Override
      public Similarity get(String name) {
        assertEquals("field", name);
        return sim;
      }
    });
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  writer.addDocument(config.build(taxoWriter, doc));
  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 9
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testDetectHierarchicalField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path", "other"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 10
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testDetectMultiValuedField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  doc.add(new FacetField("a", "path2"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 11
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testGetFacetResultsTwice() throws Exception {
  // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(new FacetField("a", "1"));
  doc.add(new FacetField("b", "1"));
  iw.addDocument(config.build(taxoWriter, doc));
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
  
  List<FacetResult> res1 = facets.getAllDims(10);
  List<FacetResult> res2 = facets.getAllDims(10);
  assertEquals("calling getFacetResults twice should return the .equals()=true result", res1, res2);

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 12
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testChildCount() throws Exception {
  // LUCENE-4885: FacetResult.numValidDescendants was not set properly by FacetsAccumulator
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();
  for (int i = 0; i < 10; i++) {
    Document doc = new Document();
    doc.add(new FacetField("a", Integer.toString(i)));
    iw.addDocument(config.build(taxoWriter, doc));
  }
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
  
  Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
  
  assertEquals(10, facets.getTopChildren(2, "a").childCount);

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 13
Source Project: lucene-solr   Source File: TestTaxonomyCombined.java    License: Apache License 2.0 6 votes vote down vote up
/**  testWriterTwice is exactly like testWriter, except that after adding
  all the categories, we add them again, and see that we get the same
  old ids again - not new categories.
 */
@Test
public void testWriterTwice() throws Exception {
  Directory indexDir = newDirectory();
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
  fillTaxonomy(tw);
  // run fillTaxonomy again - this will try to add the same categories
  // again, and check that we see the same ordinal paths again, not
  // different ones. 
  fillTaxonomy(tw);
  // Let's check the number of categories again, to see that no
  // extraneous categories were created:
  assertEquals(expectedCategories.length, tw.getSize());    
  tw.close();
  indexDir.close();
}
 
Example 14
Source Project: lucene-solr   Source File: TestTaxonomyCombined.java    License: Apache License 2.0 6 votes vote down vote up
/**  Test writing an empty index, and seeing that a reader finds in it
  the root category, and only it. We check all the methods on that
  root category return the expected results.
 */
@Test
public void testRootOnly() throws Exception {
  Directory indexDir = newDirectory();
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
  // right after opening the index, it should already contain the
  // root, so have size 1:
  assertEquals(1, tw.getSize());
  tw.close();
  TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
  assertEquals(1, tr.getSize());
  assertEquals(0, tr.getPath(0).length);
  assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParallelTaxonomyArrays().parents()[0]);
  assertEquals(0, tr.getOrdinal(new FacetLabel()));
  tr.close();
  indexDir.close();
}
 
Example 15
Source Project: lucene-solr   Source File: TestTaxonomyCombined.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNRT() throws Exception {
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  
  FacetLabel cp = new FacetLabel("a");
  writer.addCategory(cp);
  TaxonomyReader newReader = TaxonomyReader.openIfChanged(reader);
  assertNotNull("expected a new instance", newReader);
  assertEquals(2, newReader.getSize());
  assertNotSame(TaxonomyReader.INVALID_ORDINAL, newReader.getOrdinal(cp));
  reader.close();
  reader = newReader;
  
  writer.close();
  reader.close();
  
  dir.close();
}
 
Example 16
Source Project: lucene-solr   Source File: TestSearcherTaxonomyManager.java    License: Apache License 2.0 6 votes vote down vote up
public void testReplaceTaxonomyNRT() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);

  Directory taxoDir2 = newDirectory();
  DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(taxoDir2);
  tw2.close();

  SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(w, true, null, tw);
  w.addDocument(new Document());
  tw.replaceTaxonomy(taxoDir2);
  taxoDir2.close();

  expectThrows(IllegalStateException.class, () -> {
    mgr.maybeRefresh();
  });

  w.close();
  IOUtils.close(mgr, tw, taxoDir, dir);
}
 
Example 17
Source Project: lucene-solr   Source File: TestSearcherTaxonomyManager.java    License: Apache License 2.0 6 votes vote down vote up
public void testExceptionDuringRefresh() throws Exception {

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();

    IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    w.commit();
    tw.commit();

    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);

    tw.addCategory(new FacetLabel("a", "b"));
    w.addDocument(new Document());

    tw.commit();
    w.commit();

    // intentionally corrupt the taxo index:
    SegmentInfos infos = SegmentInfos.readLatestCommit(taxoDir);
    taxoDir.deleteFile(infos.getSegmentsFileName());
    expectThrows(IndexNotFoundException.class, mgr::maybeRefreshBlocking);
    IOUtils.close(w, tw, mgr, indexDir, taxoDir);
  }
 
Example 18
Source Project: lucene-solr   Source File: TestOrdinalMappingLeafReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaxonomyMergeUtils() throws Exception {
  Directory srcIndexDir = newDirectory();
  Directory srcTaxoDir = newDirectory();
  buildIndexWithFacets(srcIndexDir, srcTaxoDir, true);
  
  Directory targetIndexDir = newDirectory();
  Directory targetTaxoDir = newDirectory();
  buildIndexWithFacets(targetIndexDir, targetTaxoDir, false);
  
  IndexWriter destIndexWriter = new IndexWriter(targetIndexDir, newIndexWriterConfig(null));
  DirectoryTaxonomyWriter destTaxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
  try {
    TaxonomyMergeUtils.merge(srcIndexDir, srcTaxoDir, new MemoryOrdinalMap(), destIndexWriter, destTaxoWriter, facetConfig);
  } finally {
    IOUtils.close(destIndexWriter, destTaxoWriter);
  }
  verifyResults(targetIndexDir, targetTaxoDir);
  
  IOUtils.close(targetIndexDir, targetTaxoDir, srcIndexDir, srcTaxoDir);
}
 
Example 19
Source Project: lucene-solr   Source File: TestOrdinalMappingLeafReader.java    License: Apache License 2.0 6 votes vote down vote up
private void buildIndexWithFacets(Directory indexDir, Directory taxoDir, boolean asc) throws IOException {
  IndexWriterConfig config = newIndexWriterConfig(null);
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexDir, config);
  
  DirectoryTaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
  for (int i = 1; i <= NUM_DOCS; i++) {
    Document doc = new Document();
    for (int j = i; j <= NUM_DOCS; j++) {
      int facetValue = asc ? j: NUM_DOCS - j;
      doc.add(new FacetField("tag", Integer.toString(facetValue)));
    }
    // add a facet under default dim config
    doc.add(new FacetField("id", Integer.toString(i)));
    
    // make sure OrdinalMappingLeafReader ignores non-facet BinaryDocValues fields
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
    doc.add(new BinaryDocValuesField("cbdv", new BytesRef(Integer.toString(i*2))));
    writer.addDocument(facetConfig.build(taxonomyWriter, doc));
  }
  taxonomyWriter.commit();
  taxonomyWriter.close();
  writer.commit();
  writer.close();
}
 
Example 20
public void testNoScore() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();
  for (int i = 0; i < 4; i++) {
    Document doc = new Document();
    doc.add(new NumericDocValuesField("price", (i+1)));
    doc.add(new FacetField("a", Integer.toString(i % 2)));
    iw.addDocument(config.build(taxoWriter, doc));
  }
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  FacetsCollector sfc = new FacetsCollector();
  newSearcher(r).search(new MatchAllDocsQuery(), sfc);
  Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, DoubleValuesSource.fromLongField("price"));
  assertEquals("dim=a path=[] value=10.0 childCount=2\n  1 (6.0)\n  0 (4.0)\n", facets.getTopChildren(10, "a").toString());

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example 21
Source Project: orientdb-lucene   Source File: OLuceneFacetManager.java    License: Apache License 2.0 6 votes vote down vote up
protected void buildFacetIndexIfNeeded() throws IOException {

    if (metadata != null && metadata.containsField(FACET_FIELDS)) {
      ODatabaseDocumentInternal database = owner.getDatabase();
      Iterable<String> iterable = metadata.field(FACET_FIELDS);
      if (iterable != null) {
        Directory dir = getTaxDirectory(database);
        taxonomyWriter = new DirectoryTaxonomyWriter(dir, IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        for (String s : iterable) {
          facetField = s;
          // facetField = "facet_" + s;
          // facetDim = s;
          // config.setIndexFieldName(s, "facet_" + s);
          config.setHierarchical(s, true);
        }
      }

    }
  }
 
Example 22
Source Project: lumongo   Source File: LumongoIndex.java    License: Apache License 2.0 6 votes vote down vote up
public DirectoryTaxonomyWriter getTaxoWriter(int segmentNumber) throws IOException {

		Directory d;

		if (indexConfig.getIndexSettings().getStoreIndexOnDisk()) {
			d = MMapDirectory.open(getPathForFacetsIndex(segmentNumber));
		}
		else {
			String indexSegmentDbName = getIndexSegmentDbName(segmentNumber);
			String indexSegmentCollectionName = getIndexSegmentCollectionName(segmentNumber) + "_facets";
			MongoDirectory mongoDirectory = new MongoDirectory(mongo, indexSegmentDbName, indexSegmentCollectionName, clusterConfig.isSharded(),
					clusterConfig.getIndexBlockSize());
			d = new DistributedDirectory(mongoDirectory);
		}

		NRTCachingDirectory nrtCachingDirectory = new NRTCachingDirectory(d, 2, 10);

		return new DirectoryTaxonomyWriter(nrtCachingDirectory);
	}
 
Example 23
Source Project: linden   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method is used by the index update combiner and process an
 * intermediate form into the current intermediate form. More specifically,
 * the input intermediate forms are a single-document ram index and/or a
 * single delete term.
 * @param form  the input intermediate form
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (form.dir.ramBytesUsed() > 0 || form.taxoDir.ramBytesUsed() > 0) {
    if (writer == null) {
      createWriter();
    }

    if (facetsConfig != null) {
      DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
      // merge the taxonomies
      taxoWriter.addTaxonomy(form.taxoDir, map);
      int ordinalMap[] = map.getMap();
      DirectoryReader reader = DirectoryReader.open(form.dir);
      try {
        List<AtomicReaderContext> leaves = reader.leaves();
        int numReaders = leaves.size();
        AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
        for (int i = 0; i < numReaders; i++) {
          wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
        }
        writer.addIndexes(new MultiReader(wrappedLeaves));
      } finally {
        reader.close();
      }
    } else {
      writer.addIndexes(new Directory[] { form.dir });
    }
    numDocs++;
  }
}
 
Example 24
Source Project: linden   Source File: ShardWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
 
Example 25
@Test
public void testRecreateTaxonomy() throws Exception {
  replicator.publish(createRevision(1));
  client.updateNow();
  
  // recreate index and taxonomy
  Directory newTaxo = newDirectory();
  new DirectoryTaxonomyWriter(newTaxo).close();
  publishTaxoWriter.replaceTaxonomy(newTaxo);
  publishIndexWriter.deleteAll();
  replicator.publish(createRevision(2));
  
  client.updateNow();
  newTaxo.close();
}
 
Example 26
Source Project: lucene-solr   Source File: TestFacetsConfig.java    License: Apache License 2.0 5 votes vote down vote up
public void testAddSameDocTwice() throws Exception {
  // LUCENE-5367: this was a problem with the previous code, making sure it
  // works with the new code.
  Directory indexDir = newDirectory(), taxoDir = newDirectory();
  IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig facetsConfig = new FacetsConfig();
  Document doc = new Document();
  doc.add(new FacetField("a", "b"));
  doc = facetsConfig.build(taxoWriter, doc);
  // these two addDocument() used to fail
  indexWriter.addDocument(doc);
  indexWriter.addDocument(doc);
  indexWriter.close();
  IOUtils.close(taxoWriter);
  
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
  IndexSearcher searcher = newSearcher(indexReader);
  FacetsCollector fc = new FacetsCollector();
  searcher.search(new MatchAllDocsQuery(), fc);
  
  Facets facets = getTaxonomyFacetCounts(taxoReader, facetsConfig, fc);
  FacetResult res = facets.getTopChildren(10, "a");
  assertEquals(1, res.labelValues.length);
  assertEquals(2, res.labelValues[0].value);
  IOUtils.close(indexReader, taxoReader);
  
  IOUtils.close(indexDir, taxoDir);
}
 
Example 27
Source Project: lucene-solr   Source File: TestDrillDownQuery.java    License: Apache License 2.0 5 votes vote down vote up
public void testRequireDimensionDrillDown() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, 
                newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)));
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();

  config.setRequireDimensionDrillDown("a", true);
  config.setRequireDimensionDrillDown("b", false);

  Document doc = new Document();
  doc.add(new FacetField("a", "1"));
  doc.add(new FacetField("b", "2"));
  writer.addDocument(config.build(taxoWriter, doc));
  taxoWriter.close();

  IndexReader reader = writer.getReader();
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
  IndexSearcher searcher = newSearcher(reader);

  DrillDownQuery q = new DrillDownQuery(config);
  q.add("a", "1");
  assertEquals(1, searcher.count(q));

  q = new DrillDownQuery(config);
  q.add("a");
  assertEquals(1, searcher.count(q));

  q = new DrillDownQuery(config);
  q.add("b", "2");
  assertEquals(1, searcher.count(q));

  q = new DrillDownQuery(config);
  q.add("b");
  // no hits because we disabled dimension drill down for dimension "b":
  assertEquals(0, searcher.count(q));
  IOUtils.close(taxoReader, reader, writer, dir, taxoDir);
}
 
Example 28
Source Project: lucene-solr   Source File: TestDrillSideways.java    License: Apache License 2.0 5 votes vote down vote up
public void testScorer() throws Exception {
  // LUCENE-6001 some scorers, eg ReqExlScorer, can hit NPE if cost is called after nextDoc
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();

  // Writes facet ords to a separate directory from the
  // main index:
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);

  FacetsConfig config = new FacetsConfig();

  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(newTextField("field", "foo bar", Field.Store.NO));
  doc.add(new FacetField("Author", "Bob"));
  doc.add(new FacetField("dim", "a"));
  writer.addDocument(config.build(taxoWriter, doc));

  // NRT open
  IndexSearcher searcher = newSearcher(writer.getReader());

  // NRT open
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  DrillSideways ds = getNewDrillSideways(searcher, config, taxoReader);

  BooleanQuery.Builder bq = new BooleanQuery.Builder();
  bq.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST);
  bq.add(new TermQuery(new Term("field", "bar")), BooleanClause.Occur.MUST_NOT);
  DrillDownQuery ddq = new DrillDownQuery(config, bq.build());
  ddq.add("field", "foo");
  ddq.add("author", bq.build());
  ddq.add("dim", bq.build());
  DrillSidewaysResult r = ds.search(null, ddq, 10);
  assertEquals(0, r.hits.totalHits.value);

  writer.close();
  IOUtils.close(searcher.getIndexReader(), taxoReader, taxoWriter, dir, taxoDir);
}
 
Example 29
Source Project: lucene-solr   Source File: TestMultipleIndexFields.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDefault() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
  FacetsConfig config = getConfig();

  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  // Obtain facets results and hand-test them
  assertCorrectResults(getTaxonomyFacetCounts(tr, config, sfc));

  assertOrdinalsExist("$facets", ir);

  iw.close();
  IOUtils.close(tr, ir, tw, indexDir, taxoDir);
}
 
Example 30
Source Project: lucene-solr   Source File: TestMultipleIndexFields.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCustom() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Author", "$author");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  facetsMap.put("Author", getTaxonomyFacetCounts(tr, config, sfc, "$author"));
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);

  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$author", ir);

  iw.close();
  IOUtils.close(tr, ir, tw, indexDir, taxoDir);
}