org.apache.lucene.store.IndexOutput Java Examples

The following examples show how to use org.apache.lucene.store.IndexOutput. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCodecUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testCheckFooterInvalid() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  output.writeInt(CodecUtil.FOOTER_MAGIC);
  output.writeInt(0);
  output.writeLong(1234567); // write a bogus checksum
  output.close();

  ChecksumIndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  CodecUtil.checkHeader(input, "FooBar", 5, 5);
  assertEquals("this is the data", input.readString());
  Exception mine = new RuntimeException("fake exception");
  CorruptIndexException expected = expectThrows(CorruptIndexException.class, () -> {
    CodecUtil.checkFooter(input, mine);
  });
  assertTrue(expected.getMessage().contains("checksum failed"));
  Throwable suppressed[] = expected.getSuppressed();
  assertEquals(1, suppressed.length);
  assertEquals("fake exception", suppressed[0].getMessage());
  input.close();
}
 
Example #2
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Nightly
public void testFixedLengthHeap() throws Exception {
  // Make sure the RAM accounting is correct, i.e. if we are sorting fixed width
  // ints (4 bytes) then the heap used is really only 4 bytes per value:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    for (int i=0;i<1024*1024;i++) {
      random().nextBytes(bytes);
      w.write(bytes);
    }
    CodecUtil.writeFooter(out);
  }

  ExecutorService exec = randomExecutorServiceOrNull();
  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES, exec, TestUtil.nextInt(random(), 1, 4));
  sorter.sort(out.getName());
  if (exec != null) {
    exec.shutdownNow();
  }
  // 1 MB of ints with 4 MH heap allowed should have been sorted in a single heap partition:
  assertEquals(0, sorter.sortInfo.mergeRounds);
  dir.close();
}
 
Example #3
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMakeLockDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.obtainLock("foobar");
  });

  cfs.close();
  dir.close();
}
 
Example #4
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSyncDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.sync(Collections.singleton(testfile));
  });

  cfs.close();
  dir.close();
}
 
Example #5
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testCorruptFilesAreCaught() throws Exception {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";

  // wrong checksum
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i=0; i < 1024; i++) {
      os.writeByte((byte) i);
    }

    // write footer w/ wrong checksum
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);

    long checksum = os.getChecksum();
    os.writeLong(checksum+1);
  }

  si.setFiles(Collections.singletonList(subFile));
  Exception e = expectThrows(CorruptIndexException.class, () -> si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT));
  assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  dir.close();
}
 
Example #6
Source File: GenericRecordReader.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private static Directory copyFilesLocally(Configuration configuration, Directory dir, String table, Path shardDir,
    Path localCachePath, Collection<String> files, String segmentName) throws IOException {
  LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
  Path localShardPath = new Path(new Path(new Path(localCachePath, table), shardDir.getName()), segmentName);
  HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath, null);
  for (String name : files) {
    if (!isValidFileToCache(name)) {
      continue;
    }
    LOG.info("Valid file for local copy [{0}].", name);
    if (!isValid(localDir, dir, name)) {
      LastModified lastModified = (LastModified) dir;
      long fileModified = lastModified.getFileModified(name);

      IndexInput input = dir.openInput(name, IOContext.READONCE);
      IndexOutput output = localDir.createOutput(name, IOContext.READONCE);
      output.copyBytes(input, input.length());
      output.close();
      IndexOutput lastMod = localDir.createOutput(name + LASTMOD, IOContext.DEFAULT);
      lastMod.writeLong(fileModified);
      lastMod.close();
    }
  }
  return localDir;
}
 
Example #7
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testDeleteFileDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.deleteFile(testfile);
  });

  cfs.close();
  dir.close();
}
 
Example #8
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testLeftoverTempFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  w.close();

  IndexOutput out = dir.createTempOutput("_0", "bkd", IOContext.DEFAULT);
  String tempName = out.getName();
  out.close();
  iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  w = new IndexWriter(dir, iwc);

  // Make sure IW deleted the unref'd file:
  try {
    dir.openInput(tempName, IOContext.DEFAULT);
    fail("did not hit exception");
  } catch (FileNotFoundException | NoSuchFileException e) {
    // expected
  }
  w.close();
  dir.close();
}
 
Example #9
Source File: TestIndexInput.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testRawIndexInputRead() throws IOException {
  for (int i = 0; i < 10; i++) {
    Random random = random();
    final Directory dir = newDirectory();
    IndexOutput os = dir.createOutput("foo", newIOContext(random));
    os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
    os.close();
    IndexInput is = dir.openInput("foo", newIOContext(random));
    checkReads(is, IOException.class);
    is.close();
  
    os = dir.createOutput("bar", newIOContext(random));
    os.writeBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.length);
    os.close();
    is = dir.openInput("bar", newIOContext(random));
    checkRandomReads(is);
    is.close();
    dir.close();
  }
}
 
Example #10
Source File: TestLucene84PostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
  CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
  for (Impact impact : impacts) {
    acc.add(impact.freq, impact.norm);
  }
  try(Directory dir = newDirectory()) {
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      Lucene84SkipWriter.writeImpacts(acc, out);
    }
    try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
      byte[] b = new byte[Math.toIntExact(in.length())];
      in.readBytes(b, 0, b.length);
      List<Impact> impacts2 = Lucene84ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
      assertEquals(impacts, impacts2);
    }
  }
}
 
Example #11
Source File: ComplexVector.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * Transforms vector to cartesian form and writes vector out in dense format, truncating the
 * vectors to the assigned dimensionality
 */
public void writeToLuceneStream(IndexOutput outputStream, int k) {
  toCartesian();
  for (int i = 0; i < k * 2; ++i) {
    try {
      outputStream.writeInt(Float.floatToIntBits(coordinates[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
    
    
    /* DORMANT CODE!
  assert(opMode != MODE.POLAR_SPARSE);
  if (opMode == MODE.CARTESIAN) {
    cartesianToDensePolar();
  }
  for (int i = 0; i < dimension; ++i) {
    try {
      outputStream.writeInt((int)(phaseAngles[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
     */
}
 
Example #12
Source File: PermutationVector.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
/**
 * Writes vector out in dense format.
 */
public void writeToLuceneStream(IndexOutput outputStream) {
  int[] coordsToWrite;
  
    coordsToWrite = coordinates;
  
  for (int i = 0; i < dimension; ++i) {
    try {
      outputStream.writeInt((coordsToWrite[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 
Example #13
Source File: CacheIndexInputTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test2() throws IOException {
  Cache cache = getCache();
  RAMDirectory directory = new RAMDirectory();
  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  writeRandomData(size, random, output);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = new CacheIndexInput(null, name, input.clone(), cache);
  readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  readRandomDataShort(input, testInput, random, sampleSize);
  readRandomDataInt(input, testInput, random, sampleSize);
  readRandomDataLong(input, testInput, random, sampleSize);
  testInput.close();
  input.close();
  directory.close();
}
 
Example #14
Source File: BKDWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
OneDimensionBKDWriter(IndexOutput metaOut, IndexOutput indexOut, IndexOutput dataOut) {
  if (numIndexDims != 1) {
    throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
  }
  if (pointCount != 0) {
    throw new IllegalStateException("cannot mix add and merge");
  }

  // Catch user silliness:
  if (finished == true) {
    throw new IllegalStateException("already finished");
  }

  // Mark that we already finished:
  finished = true;

  this.metaOut = metaOut;
  this.indexOut = indexOut;
  this.dataOut = dataOut;
  this.dataStartFP = dataOut.getFilePointer();

  lastPackedValue = new byte[packedBytesLength];
}
 
Example #15
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testPositionNotZero() throws IOException {
  final int BLOCKS = 10;
  final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable

  BitSet set = createSetWithRandomBlocks(BLOCKS);
  try (Directory dir = newDirectory()) {
    final int cardinality = set.cardinality();
    int jumpTableEntryCount;
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      jumpTableEntryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
    }
    try (IndexInput fullInput = dir.openInput("foo", IOContext.DEFAULT)) {
      IndexInput blockData =
          IndexedDISI.createBlockSlice(fullInput, "blocks", 0, fullInput.length(), jumpTableEntryCount);
      blockData.seek(random().nextInt((int) blockData.length()));

      RandomAccessInput jumpTable = IndexedDISI.createJumpTable(fullInput, 0, fullInput.length(), jumpTableEntryCount);
      IndexedDISI disi = new IndexedDISI(blockData, jumpTable, jumpTableEntryCount, denseRankPower, cardinality);
      // This failed at some point during LUCENE-8585 development as it did not reset the slice position
      disi.advanceExact(BLOCKS*65536-1);
    }
  }
}
 
Example #16
Source File: BlockWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected BlockWriter(IndexOutput blockOutput, int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) {
  assert blockOutput != null;
  assert targetNumBlockLines > 0;
  assert deltaNumLines >= 0;
  assert deltaNumLines < targetNumBlockLines;
  this.blockOutput = blockOutput;
  this.targetNumBlockLines = targetNumBlockLines;
  this.deltaNumLines = deltaNumLines;
  this.blockEncoder = blockEncoder;

  this.blockLines = new ArrayList<>(targetNumBlockLines);
  this.blockHeaderWriter = createBlockHeaderSerializer();
  this.blockLineWriter = createBlockLineSerializer();
  this.termStateSerializer = createDeltaBaseTermStateSerializer();

  this.blockLinesWriteBuffer = ByteBuffersDataOutput.newResettableInstance();
  this.termStatesWriteBuffer = ByteBuffersDataOutput.newResettableInstance();
  this.blockWriteBuffer = ByteBuffersDataOutput.newResettableInstance();

  this.reusableBlockHeader = new BlockHeader();
  this.scratchBytesRef = new BytesRef();
}
 
Example #17
Source File: BinaryVectorTest.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testGenerateRandomVectorWriteAndRead() {
  Random random = new Random(0);

  Vector vector = VectorFactory.generateRandomVector(VectorType.BINARY, 64, 2, random);
  // The exact string depends on fail Java's implementation of Random, so we only check for length.
  String vectorString = vector.writeToString();
  assertEquals(64, vectorString.length());

  RAMDirectory directory = new RAMDirectory();
  try {
    IndexOutput indexOutput = directory.createOutput("binaryvectors.bin", IOContext.DEFAULT);
    vector.writeToLuceneStream(indexOutput);
    indexOutput.close();
    IndexInput indexInput = directory.openInput("binaryvectors.bin", IOContext.DEFAULT);
    Vector vector2 = VectorFactory.createZeroVector(VectorType.BINARY, 64);
    assertEquals("0000000000000000000000000000000000000000000000000000000000000000", vector2.writeToString());
    vector2.readFromLuceneStream(indexInput);
    assertEquals(vectorString, vector2.writeToString());
  } catch (IOException e) {
    e.printStackTrace();
    fail();
  }
  directory.close();
}
 
Example #18
Source File: RecoveryTarget.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
protected void closeInternal() {
    try {
        // clean open index outputs
        Iterator<Entry<String, IndexOutput>> iterator = openIndexOutputs.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry<String, IndexOutput> entry = iterator.next();
            logger.trace("closing IndexOutput file [{}]", entry.getValue());
            try {
                entry.getValue().close();
            } catch (Exception e) {
                logger.debug(() -> new ParameterizedMessage("error while closing recovery output [{}]", entry.getValue()), e);
            }
            iterator.remove();
        }
        // trash temporary files
        for (String file : tempFileNames.keySet()) {
            logger.trace("cleaning temporary file [{}]", file);
            store.deleteQuiet(file);
        }
    } finally {
        // free store. increment happens in constructor
        fileChunkWriters.clear();
        store.decRef();
        indexShard.recoveryStats().decCurrentAsTarget();
        closedLatch.countDown();
    }
}
 
Example #19
Source File: SimpleTextUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static void writeChecksum(IndexOutput out, BytesRefBuilder scratch) throws IOException {
  // Pad with zeros so different checksum values use the
  // same number of bytes
  // (BaseIndexFileFormatTestCase.testMergeStability cares):
  String checksum = String.format(Locale.ROOT, "%020d", out.getChecksum());
  write(out, CHECKSUM);
  write(out, checksum, scratch);
  writeNewline(out);
}
 
Example #20
Source File: HdfsDirectoryTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testCantOverrideFiles() throws IOException {
  try (IndexOutput out = directory.createOutput("foo", IOContext.DEFAULT)) {
    out.writeByte((byte) 42);
  }
  expectThrows(FileAlreadyExistsException.class,
      () -> directory.createOutput("foo", IOContext.DEFAULT));
}
 
Example #21
Source File: BlockDirectory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public IndexOutput createOutput(String name, IOContext context)
    throws IOException {
  final IndexOutput dest = super.createOutput(name, context);
  if (useWriteCache(name, context)) {
    return new CachedIndexOutput(this, dest, blockSize, name, cache, blockSize);
  }
  return dest;
}
 
Example #22
Source File: CompletionFieldsConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Stores the built FST in <code>output</code>
 * Returns true if there was anything stored, false otherwise
 */
public boolean finish(IndexOutput output) throws IOException {
  boolean stored = builder.store(output);
  assert stored || docCount == 0 : "the FST is null but docCount is != 0 actual value: [" + docCount + "]";
  if (docCount == 0) {
    minWeight = 0;
  }
  return stored;
}
 
Example #23
Source File: TestCodecUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testSegmentHeaderLength() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeIndexHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
  output.writeString("this is the data");
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  input.seek(CodecUtil.indexHeaderLength("FooBar", "xyz"));
  assertEquals("this is the data", input.readString());
  input.close();
}
 
Example #24
Source File: TestCodecUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testChecksumEntireFile() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  CodecUtil.writeFooter(output);
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  CodecUtil.checksumEntireFile(input);
  input.close();
}
 
Example #25
Source File: RecoveryStatus.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an {@link org.apache.lucene.store.IndexOutput} for the given file name. Note that the
 * IndexOutput actually point at a temporary file.
 * <p>
 * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput
 * at a later stage
 */
public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException {
    ensureRefCount();
    String tempFileName = getTempNameForFile(fileName);
    if (tempFileNames.containsKey(tempFileName)) {
        throw new IllegalStateException("output for file [" + fileName + "] has already been created");
    }
    // add first, before it's created
    tempFileNames.put(tempFileName, fileName);
    IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT);
    openIndexOutputs.put(fileName, indexOutput);
    return indexOutput;
}
 
Example #26
Source File: CustomBufferedIndexInput.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Flushes the in-memory bufer to the given output, copying at most
 * <code>numBytes</code>.
 * <p>
 * <b>NOTE:</b> this method does not refill the buffer, however it does
 * advance the buffer position.
 * 
 * @return the number of bytes actually flushed from the in-memory buffer.
 */
protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
  int toCopy = bufferLength - bufferPosition;
  if (toCopy > numBytes) {
    toCopy = (int) numBytes;
  }
  if (toCopy > 0) {
    out.writeBytes(buffer, bufferPosition, toCopy);
    bufferPosition += toCopy;
  }
  return toCopy;
}
 
Example #27
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testCheckIntegrity() throws IOException {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i = 0; i < 1024; i++) {
      os.writeByte((byte) i);
    }
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);
    long checksum = os.getChecksum();
    os.writeLong(checksum);
  }

  si.setFiles(Collections.singletonList(subFile));
  
  FileTrackingDirectoryWrapper writeTrackingDir = new FileTrackingDirectoryWrapper(dir);
  si.getCodec().compoundFormat().write(writeTrackingDir, si, IOContext.DEFAULT);
  final Set<String> createdFiles = writeTrackingDir.getFiles();

  ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
  CompoundDirectory compoundDir = si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.READ);
  compoundDir.checkIntegrity();
  Map<String,FixedBitSet> readBytes = readTrackingDir.getReadBytes();
  assertEquals(createdFiles, readBytes.keySet());
  for (Map.Entry<String, FixedBitSet> entry : readBytes.entrySet()) {
    final String file = entry.getKey();
    final FixedBitSet set = entry.getValue().clone();
    set.flip(0, set.length());
    final int next = set.nextSetBit(0);
    assertEquals("Byte at offset " + next + " of " + file + " was not read", DocIdSetIterator.NO_MORE_DOCS, next);
  }
  compoundDir.close();
  dir.close();
}
 
Example #28
Source File: TestPackedInts.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testSave() throws IOException {
  final int valueCount = TestUtil.nextInt(random(), 1, 2048);
  for (int bpv = 1; bpv <= 64; ++bpv) {
    final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
    final Directory directory = new ByteBuffersDirectory();
    List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
    for (PackedInts.Mutable mutable : packedInts) {
      for (int i = 0; i < mutable.size(); ++i) {
        mutable.set(i, random().nextInt(maxValue));
      }

      IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
      mutable.save(out);
      out.close();

      IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
      PackedInts.Reader reader = PackedInts.getReader(in);
      assertEquals(valueCount, reader.size());
      if (mutable instanceof Packed64SingleBlock) {
        // make sure that we used the right format so that the reader has
        // the same performance characteristics as the mutable that has been
        // serialized
        assertTrue(reader instanceof Packed64SingleBlock);
      } else {
        assertFalse(reader instanceof Packed64SingleBlock);
      }
      for (int i = 0; i < valueCount; ++i) {
        assertEquals(mutable.get(i), reader.get(i));
      }
      in.close();
      directory.deleteFile("packed-ints.bin");
    }
    directory.close();
  }
}
 
Example #29
Source File: SimpleTextLiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
  int size = bits.length();
  BytesRefBuilder scratch = new BytesRefBuilder();
  
  String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getNextDelGen());
  IndexOutput out = null;
  boolean success = false;
  try {
    out = dir.createOutput(fileName, context);
    SimpleTextUtil.write(out, SIZE);
    SimpleTextUtil.write(out, Integer.toString(size), scratch);
    SimpleTextUtil.writeNewline(out);
    
    for (int i = 0; i < size; ++i) {
      if (bits.get(i)) {
        SimpleTextUtil.write(out, DOC);
        SimpleTextUtil.write(out, Integer.toString(i), scratch);
        SimpleTextUtil.writeNewline(out);
      }
    }
    
    SimpleTextUtil.write(out, END);
    SimpleTextUtil.writeNewline(out);
    SimpleTextUtil.writeChecksum(out, scratch);
    success = true;
  } finally {
    if (success) {
      IOUtils.close(out);
    } else {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 
Example #30
Source File: CacheDirectoryTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Test
public void test2() throws IOException {
  IndexOutput output = _cacheDirectory.createOutput("test.file", IOContext.DEFAULT);
  byte[] buf = new byte[9000];
  for (int i = 0; i < buf.length; i++) {
    buf[i] = (byte) i;
  }
  output.writeBytes(buf, buf.length);
  output.close();

  IndexInput input = _cacheDirectory.openInput("test.file", IOContext.DEFAULT);
  assertEquals(9000, input.length());
  input.close();
}