org.apache.lucene.store.RAMDirectory Java Examples

The following examples show how to use org.apache.lucene.store.RAMDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LuceneInMemorySearchIntegrationTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
public void givenSortFieldWhenSortedThenCorrect() {
    InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
    inMemoryLuceneIndex.indexDocument("Ganges", "River in India");
    inMemoryLuceneIndex.indexDocument("Mekong", "This river flows in south Asia");
    inMemoryLuceneIndex.indexDocument("Amazon", "Rain forest river");
    inMemoryLuceneIndex.indexDocument("Rhine", "Belongs to Europe");
    inMemoryLuceneIndex.indexDocument("Nile", "Longest River");

    Term term = new Term("body", "river");
    Query query = new WildcardQuery(term);

    SortField sortField = new SortField("title", SortField.Type.STRING_VAL, false);
    Sort sortByTitle = new Sort(sortField);

    List<Document> documents = inMemoryLuceneIndex.searchIndex(query, sortByTitle);
    Assert.assertEquals(4, documents.size());
    Assert.assertEquals("Amazon", documents.get(0).getField("title").stringValue());
}
 
Example #2
Source File: NGramTestSetup.java    From uyuni with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Creates an index in RAM
 * */
public void setUp() throws Exception {
    super.setUp();
    initItems();
    this.stanDir = new RAMDirectory();
    IndexWriter stanWriter = new IndexWriter(this.stanDir, new StandardAnalyzer(), true);

    this.ngramDir = new RAMDirectory();
    IndexWriter ngramWriter = new IndexWriter(this.ngramDir, new NGramAnalyzer(min_ngram, max_ngram), true);

    for (Map<String, String> item: items) {
        String name = item.get("name");
        String descp = item.get("description");
        Document doc = new Document();
        doc.add(new Field("name", name, Field.Store.YES, Field.Index.TOKENIZED));
        doc.add(new Field("description", descp, Field.Store.YES,
                Field.Index.TOKENIZED));
        stanWriter.addDocument(doc);
        ngramWriter.addDocument(doc);
    }
    stanWriter.close();
    ngramWriter.close();
}
 
Example #3
Source File: TestLucene.java    From RedisDirectory with Apache License 2.0 6 votes vote down vote up
public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}
 
Example #4
Source File: LuceneIndex.java    From netbeans with Apache License 2.0 6 votes vote down vote up
private CleanReference(final RAMDirectory[] dir) {
    super (dir, BaseUtilities.activeReferenceQueue());
    final IndexCacheFactory.RAMController c = IndexCacheFactory.getDefault().getRAMController();
    final boolean doHardRef = !c.isFull();
    if (doHardRef) {
        this.hardRef = dir;
        long _size = dir[0].sizeInBytes();
        size.set(_size);
        c.acquire(_size);
    }
    LOGGER.log(Level.FINEST, "Caching index: {0} cache policy: {1}",    //NOI18N
    new Object[]{
        folder.getAbsolutePath(),
        cachePolicy.getSystemName()
    });
}
 
Example #5
Source File: IndexInfo.java    From alfresco-repository with GNU Lesser General Public License v3.0 6 votes vote down vote up
private IndexReader buildReferenceCountingIndexReader(String id, long size) throws IOException
{
    IndexReader reader;
    File location = new File(indexDirectory, id).getCanonicalFile();
    double folderSize = getSizeInMb(location);
    if (IndexReader.indexExists(location))
    {
        if ((size < maxDocsForInMemoryIndex) && (folderSize < maxRamInMbForInMemoryIndex))
        {
            RAMDirectory rd = new RAMDirectory(location);
            reader = IndexReader.open(rd);
        }
        else
        {
            reader = IndexReader.open(location);
        }
    }
    else
    {
        reader = IndexReader.open(emptyIndex);
    }
    reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, reader, true, config);
    return reader;
}
 
Example #6
Source File: SpatialHelperTest.java    From geode-examples with Apache License 2.0 6 votes vote down vote up
@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 
Example #7
Source File: AnchorIndexer.java    From tagme with Apache License 2.0 6 votes vote down vote up
private IndexSearcher openWikipediaIndex(String lang) throws IOException{
	
	
	File indexDir = RepositoryDirs.WIKIPEDIA.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	if (indexSize < maxMemory * GAP_FACTOR){
		
		log.info("MaxMemory is enough, loading Wikipedia index...");
		IndexReader r = IndexReader.open(new RAMDirectory(FSDirectory.open(indexDir)), true);
		log.info("WikipediaIndex loaded.");
		return new IndexSearcher(r);
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load WikipediaIndex (about "+indexSize/1000000+"Mb)");
		return Indexes.getSearcher(RepositoryDirs.WIKIPEDIA.getPath(lang));
	}
}
 
Example #8
Source File: LuceneInMemorySearchIntegrationTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
public void givenBooleanQueryWhenFetchedDocumentThenCorrect() {
    InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
    inMemoryLuceneIndex.indexDocument("Destination", "Las Vegas singapore car");
    inMemoryLuceneIndex.indexDocument("Commutes in singapore", "Bus Car Bikes");

    Term term1 = new Term("body", "singapore");
    Term term2 = new Term("body", "car");

    TermQuery query1 = new TermQuery(term1);
    TermQuery query2 = new TermQuery(term2);

    BooleanQuery booleanQuery = new BooleanQuery.Builder().add(query1, BooleanClause.Occur.MUST)
            .add(query2, BooleanClause.Occur.MUST).build();

    List<Document> documents = inMemoryLuceneIndex.searchIndex(booleanQuery);
    Assert.assertEquals(1, documents.size());
}
 
Example #9
Source File: TestMixedDirectory.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 
Example #10
Source File: LindenCoreImpl.java    From linden with Apache License 2.0 6 votes vote down vote up
public Directory createTaxoIndexDirectory(String directory, LindenConfig.IndexType indexType) throws IOException {
  switch (indexType) {
    case RAM:
      return new RAMDirectory();
    default:
      Preconditions.checkNotNull(directory, "index directory can not be null");
      return new NRTCachingDirectory(FSDirectory.open(new File(directory + ".taxonomy")),
                                     maxMergeSizeMB, maxCachedMB);
  }
}
 
Example #11
Source File: CodePatternSearcher.java    From SnowGraph with Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example #12
Source File: SpatialHelperTest.java    From geode-examples with Apache License 2.0 6 votes vote down vote up
@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 
Example #13
Source File: FullTextIndex.java    From jease with GNU General Public License v3.0 6 votes vote down vote up
public FullTextIndex() {
    try {
        objects = new ArrayList<>();

        Analyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig config =  new IndexWriterConfig(new LimitTokenCountAnalyzer(analyzer, Integer.MAX_VALUE));

        indexDirectory = new RAMDirectory();
        indexWriter = new IndexWriter(indexDirectory, config);

        queryParser = new QueryParser("text", analyzer);
        queryParser.setDefaultOperator(QueryParser.AND_OPERATOR);

        fulltext = new TextField("text", "", Field.Store.NO);

        // Used as base-set for a NOT-Query
        Field inverse = new TextField("true", "yes", Field.Store.NO);

        document = new Document();
        document.add(fulltext);
        document.add(inverse);
    } catch (Exception e) {
        throw new RuntimeException(e.getMessage(), e);
    }
}
 
Example #14
Source File: LuceneUtils.java    From modernmt with Apache License 2.0 6 votes vote down vote up
public static Map<String, Float> getTermFrequencies(Analyzer analyzer, Document document, String fieldName) throws IOException {
    Directory directory = new RAMDirectory();
    IndexWriter writer = null;
    IndexReader reader = null;

    try {
        // Writing document in RAM
        IndexWriterConfig indexConfig = new IndexWriterConfig(Version.LUCENE_4_10_4, analyzer);
        indexConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
        writer = new IndexWriter(directory, indexConfig);
        writer.addDocument(document);
        IOUtils.closeQuietly(writer);


        reader = DirectoryReader.open(directory);
        return getTermFrequencies(reader, 0, fieldName);
    } finally {
        IOUtils.closeQuietly(writer);
        IOUtils.closeQuietly(reader);
        IOUtils.closeQuietly(directory);
    }
}
 
Example #15
Source File: ComplexVectorTest.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testReadWrite() {
  Vector v1 = new ComplexVector(new short[] { -1, 8000, 16000 });
  RAMDirectory directory = new RAMDirectory();
  try {
    IndexOutput indexOutput = directory.createOutput("complexvectors.bin", IOContext.DEFAULT);
    v1.writeToLuceneStream(indexOutput);
    indexOutput.close();

    IndexInput indexInput = directory.openInput("complexvectors.bin", IOContext.DEFAULT);
    ComplexVector cv2 = new ComplexVector(3, Mode.POLAR_SPARSE);
    cv2.readFromLuceneStream(indexInput);
    assertFloatArrayEquals(
        new float[] {0, 0, -0.997290f, 0.073564f, 0.989176f, -0.1467304f},
        cv2.getCoordinates(), TOL);
  } catch (IOException e) {
    e.printStackTrace();
    fail();
  }
  directory.close();
}
 
Example #16
Source File: BinaryVectorTest.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testGenerateRandomVectorWriteAndRead() {
  Random random = new Random(0);

  Vector vector = VectorFactory.generateRandomVector(VectorType.BINARY, 64, 2, random);
  // The exact string depends on fail Java's implementation of Random, so we only check for length.
  String vectorString = vector.writeToString();
  assertEquals(64, vectorString.length());

  RAMDirectory directory = new RAMDirectory();
  try {
    IndexOutput indexOutput = directory.createOutput("binaryvectors.bin", IOContext.DEFAULT);
    vector.writeToLuceneStream(indexOutput);
    indexOutput.close();
    IndexInput indexInput = directory.openInput("binaryvectors.bin", IOContext.DEFAULT);
    Vector vector2 = VectorFactory.createZeroVector(VectorType.BINARY, 64);
    assertEquals("0000000000000000000000000000000000000000000000000000000000000000", vector2.writeToString());
    vector2.readFromLuceneStream(indexInput);
    assertEquals(vectorString, vector2.writeToString());
  } catch (IOException e) {
    e.printStackTrace();
    fail();
  }
  directory.close();
}
 
Example #17
Source File: VocabularyAnalyzerTest.java    From SciGraph with Apache License 2.0 6 votes vote down vote up
@Before
public void setupIndex() throws Exception {
  Directory dir = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(new VocabularyIndexAnalyzer());
  try (IndexWriter writer = new IndexWriter(dir, conf)) {
    addDoc(writer, "hippocampus");
    addDoc(writer, "hippocampal structures");
    addDoc(writer, "structure of the hippocampus");
    addDoc(writer, "formation");
    writer.commit();
  }

  IndexReader reader = DirectoryReader.open(dir);
  searcher = new IndexSearcher(reader);
  parser = new QueryParser(NodeProperties.LABEL, new VocabularyQueryAnalyzer());
}
 
Example #18
Source File: TestLuceneUnsortedIntTermDocIterator.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleTerm() throws IOException {
    RAMDirectory d = new RAMDirectory();
    IndexWriter w = new IndexWriter(d, null, true, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    doc.add(new Field("int", "1", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
    w.addDocument(doc);
    w.close();

    IndexReader r = IndexReader.open(d);
    LuceneUnsortedIntTermDocIterator iter = LuceneUnsortedIntTermDocIterator.create(r, "int");
    assertTrue(iter.nextTerm());
    assertEquals(1, iter.term());
    int[] docs = new int[2];
    assertEquals(1, iter.nextDocs(docs));
    assertEquals(0, docs[0]);
    assertFalse(iter.nextTerm());
    r.close();
}
 
Example #19
Source File: MutatableActionTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testReplaceRow() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  DirectoryReader reader = getIndexReader(directory);
  IndexWriter writer = new IndexWriter(directory, _conf.clone());
  assertEquals(0, reader.numDocs());

  Row row = genRow();
  _action.replaceRow(row);
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(1, reader.numDocs());

  Row row2 = new Row(row);
  List<Column> cols = new ArrayList<Column>();
  cols.add(new Column("n", "v"));
  row2.addToRecords(new Record("1", "fam", cols));

  _action.replaceRow(row2);
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(2, reader.numDocs());
}
 
Example #20
Source File: MutatableActionTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteRecord() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  DirectoryReader reader = getIndexReader(directory);
  IndexWriter writer = new IndexWriter(directory, _conf.clone());
  assertEquals(0, reader.numDocs());

  Row row = genRow();
  List<Column> cols = new ArrayList<Column>();
  cols.add(new Column("n", "v"));
  row.addToRecords(new Record("1", "fam", cols));

  _action.replaceRow(row);
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(2, reader.numDocs());

  _action.deleteRecord(row.getId(), "1");
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(1, reader.numDocs());
}
 
Example #21
Source File: MutatableActionTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteRow() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  DirectoryReader reader = getIndexReader(directory);
  IndexWriter writer = new IndexWriter(directory, _conf.clone());
  assertEquals(0, reader.numDocs());

  Row row = genRow();
  _action.replaceRow(row);
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(1, reader.numDocs());

  _action.deleteRow(row.getId());
  _action.performMutate(getSearcher(reader, directory), writer);
  reader = commitAndReopen(reader, writer);
  assertEquals(0, reader.numDocs());
}
 
Example #22
Source File: BlurUtilsTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private IndexReader getReader() throws CorruptIndexException, LockObtainFailedException, IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f1", Store.YES));

  Document doc1 = new Document();
  doc.add(new StringField("a", "b", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #23
Source File: BlurUtilsTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #24
Source File: Blur024CodecTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testDocValuesFormat() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
  conf.setCodec(new Blur024Codec());
  IndexWriter writer = new IndexWriter(directory, conf);

  Document doc = new Document();
  doc.add(new StringField("f", "v", Store.YES));
  doc.add(new SortedDocValuesField("f", new BytesRef("v")));
  writer.addDocument(doc);

  writer.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  AtomicReaderContext context = reader.leaves().get(0);
  AtomicReader atomicReader = context.reader();
  SortedDocValues sortedDocValues = atomicReader.getSortedDocValues("f");
  assertTrue(sortedDocValues.getClass().getName().startsWith(DiskDocValuesProducer.class.getName()));

  reader.close();
}
 
Example #25
Source File: CacheIndexInputTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test1() throws IOException {
  RAMDirectory directory = new RAMDirectory();

  String name = "test";

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  byte[] bs = "hello world".getBytes();
  output.writeBytes(bs, bs.length);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  Cache cache = getCache();
  CacheIndexInput cacheInput = new CacheIndexInput(null, name, input, cache);
  byte[] buf = new byte[bs.length];
  cacheInput.readBytes(buf, 0, buf.length);
  cacheInput.close();

  assertArrayEquals(bs, buf);
  directory.close();
}
 
Example #26
Source File: CacheIndexInputTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test2() throws IOException {
  Cache cache = getCache();
  RAMDirectory directory = new RAMDirectory();
  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  writeRandomData(size, random, output);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = new CacheIndexInput(null, name, input.clone(), cache);
  readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  readRandomDataShort(input, testInput, random, sampleSize);
  readRandomDataInt(input, testInput, random, sampleSize);
  readRandomDataLong(input, testInput, random, sampleSize);
  testInput.close();
  input.close();
  directory.close();
}
 
Example #27
Source File: CacheIndexOutputTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test1() throws IOException {
  Random random = new Random(seed);
  RAMDirectory directory = new RAMDirectory();

  Cache cache = CacheIndexInputTest.getCache();
  CacheIndexOutput indexOutput = new CacheIndexOutput(null, "test", cache, directory, IOContext.DEFAULT);
  indexOutput.writeByte((byte) 1);
  indexOutput.writeByte((byte) 2);
  byte[] b = new byte[16000];
  random.nextBytes(b);
  indexOutput.writeBytes(b, 16000);
  indexOutput.close();

  IndexInput input = directory.openInput("test", IOContext.DEFAULT);
  assertEquals(16002, input.length());
  assertEquals(1, input.readByte());
  assertEquals(2, input.readByte());

  byte[] buf = new byte[16000];
  input.readBytes(buf, 0, 16000);
  input.close();
  assertArrayEquals(b, buf);
  directory.close();
}
 
Example #28
Source File: CacheIndexOutputTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test2() throws IOException {
  Cache cache = CacheIndexInputTest.getCache();
  RAMDirectory directory = new RAMDirectory();
  RAMDirectory directory2 = new RAMDirectory();

  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  CacheIndexOutput cacheIndexOutput = new CacheIndexOutput(null, name, cache, directory2, IOContext.DEFAULT);
  CacheIndexInputTest.writeRandomData(size, random, output, cacheIndexOutput);
  output.close();
  cacheIndexOutput.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = directory2.openInput(name, IOContext.DEFAULT);
  CacheIndexInputTest.readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  testInput.close();
  input.close();
  directory.close();
  directory2.close();
}
 
Example #29
Source File: BaseDirectoryTestSuite.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testWrites() throws IOException {
  int i = 0;
  try {
    Set<String> names = new HashSet<String>();
    for (; i < 10; i++) {
      Directory fsDir = new RAMDirectory();
      String name = getName();
      System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" + names.contains(name) + "]");
      names.add(name);
      createFile(name, fsDir, directory);
      assertInputsEquals(name, fsDir, directory);
      fsDir.close();
    }
  } catch (Exception e) {
    e.printStackTrace();
    fail("Test failed with seed [" + seed + "] on pass [" + i + "]");
  }
}
 
Example #30
Source File: IndexFileBitSetTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws IOException {
  Random random = new Random(_seed);
  int numBits = random.nextInt(10000000);
  FixedBitSet fixedBitSet = new FixedBitSet(numBits);
  populate(random, numBits, fixedBitSet);
  String id = "id";
  String segmentName = "seg1";
  RAMDirectory directory = new RAMDirectory();
  IndexFileBitSet indexFileBitSet = new IndexFileBitSet(numBits, id, segmentName, directory);
  assertFalse(indexFileBitSet.exists());
  indexFileBitSet.create(fixedBitSet.iterator());
  indexFileBitSet.load();
  checkEquals(fixedBitSet.iterator(), indexFileBitSet.iterator(), numBits);
  indexFileBitSet.close();
  
  String[] listAll = directory.listAll();
  for (String s : listAll) {
    System.out.println(s + " " + directory.fileLength(s));
  }
}