Java Code Examples for org.apache.lucene.store.Directory#createOutput()

/** simple encode/decode */
public void testSimple() throws Exception {
  Directory dir = newDirectory();
  int bitsPerValue = DirectWriter.bitsRequired(2);
  IndexOutput output = dir.createOutput("foo", IOContext.DEFAULT);
  DirectWriter writer = DirectWriter.getInstance(output, 5, bitsPerValue);
  writer.add(1);
  writer.add(0);
  writer.add(2);
  writer.add(1);
  writer.add(2);
  writer.finish();
  output.close();
  IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
  LongValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
  assertEquals(1, reader.get(0));
  assertEquals(0, reader.get(1));
  assertEquals(2, reader.get(2));
  assertEquals(1, reader.get(3));
  assertEquals(2, reader.get(4));
  input.close();
  dir.close();
}
 

@Override
public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
  long gen = info.getNextDelGen();
  String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
  int delCount = 0;
  try (IndexOutput output = dir.createOutput(name, context)) {
    CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
    final int longCount = FixedBitSet.bits2words(bits.length());
    for (int i = 0; i < longCount; ++i) {
      long currentBits = 0;
      for (int j = i << 6, end = Math.min(j + 63, bits.length() - 1); j <= end; ++j) {
        if (bits.get(j)) {
          currentBits |= 1L << j; // mod 64
        } else {
          delCount += 1;
        }
      }
      output.writeLong(currentBits);
    }
    CodecUtil.writeFooter(output);
  }
  if (delCount != info.getDelCount() + newDelCount) {
    throw new CorruptIndexException("bits.deleted=" + delCount + 
        " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name);
  }
}
 

private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
  final int cardinality = set.cardinality();
  final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
  long length;
  int jumpTableentryCount;
  try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
    jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
  }

  try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
    BitSetIterator disi2 = new BitSetIterator(set, cardinality);
    int doc = disi2.docID();
    int index = 0;
    while (doc < cardinality) {
      doc = disi2.nextDoc();
      index++;
    }

    IndexedDISI disi = new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
    // Advance 1 docID beyond end
    assertFalse("There should be no set bit beyond the valid docID range", disi.advanceExact(set.length()));
    disi.advance(doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
    assertEquals("The index when advancing beyond the last defined docID should be correct",
        index, disi.index()+1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
  }
}
 

public void testSingleValue() throws Exception {
  for (int bitsPerValue = 1; bitsPerValue <= 64; ++bitsPerValue) {
    Directory dir = newDirectory();
    IndexOutput out = dir.createOutput("out", newIOContext(random()));
    PackedInts.Writer w = PackedInts.getWriter(out, 1, bitsPerValue, PackedInts.DEFAULT);
    long value = 17L & PackedInts.maxValue(bitsPerValue);
    w.add(value);
    w.finish();
    final long end = out.getFilePointer();
    out.close();

    IndexInput in = dir.openInput("out", newIOContext(random()));
    Reader reader = PackedInts.getReader(in);
    String msg = "Impl=" + w.getClass().getSimpleName() + ", bitsPerValue=" + bitsPerValue;
    assertEquals(msg, 1, reader.size());
    assertEquals(msg, value, reader.get(0));
    assertEquals(msg, end, in.getFilePointer());
    in.close();

    dir.close();
  }
}
 

public void testDoubleClose() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  assertEquals(1, cfs.listAll().length);
  cfs.close();
  cfs.close(); // second close should not throw exception
  dir.close();
}
 

public void testPassIOContext() throws IOException {
  final String testfile = "_123.test";
  final IOContext myContext = new IOContext();

  Directory dir = new FilterDirectory(newDirectory()) {
    @Override
    public IndexOutput createOutput(String name, IOContext context) throws IOException {
      assertSame(myContext, context);
      return super.createOutput(name, context);
    }
  };
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, myContext)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, myContext);
  dir.close();
}
 

public void testLargeCFS() throws IOException {   
  final String testfile = "_123.test";
  IOContext context = new IOContext(new FlushInfo(0, 512*1024*1024));

  Directory dir = new NRTCachingDirectory(newFSDirectory(createTempDir()), 2.0, 25.0);

  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, context)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    byte[] bytes = new byte[512];
    for(int i=0;i<1024*1024;i++) {
      out.writeBytes(bytes, 0, bytes.length);
    }
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, context);

  dir.close();
}
 

public void testMissingCodecHeadersAreCaught() throws Exception {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";

  // missing codec header
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    for (int i=0; i < 1024; i++) {
      os.writeByte((byte) i);
    }
  }

  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.singletonList(subFile));
  Exception e = expectThrows(CorruptIndexException.class, () -> si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT));
  assertTrue(e.getMessage().contains("codec header mismatch"));
  dir.close();
}
 

public void testRenameFileDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.rename(testfile, "bogus");
  });

  cfs.close();
  dir.close();
}
 

public void testMakeLockDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.obtainLock("foobar");
  });

  cfs.close();
  dir.close();
}
 

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
    }
    CodecUtil.writeFooter(output);
  }
}
 

public void testCheckIntegrity() throws IOException {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i = 0; i < 1024; i++) {
      os.writeByte((byte) i);
    }
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);
    long checksum = os.getChecksum();
    os.writeLong(checksum);
  }

  si.setFiles(Collections.singletonList(subFile));
  
  FileTrackingDirectoryWrapper writeTrackingDir = new FileTrackingDirectoryWrapper(dir);
  si.getCodec().compoundFormat().write(writeTrackingDir, si, IOContext.DEFAULT);
  final Set<String> createdFiles = writeTrackingDir.getFiles();

  ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
  CompoundDirectory compoundDir = si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.READ);
  compoundDir.checkIntegrity();
  Map<String,FixedBitSet> readBytes = readTrackingDir.getReadBytes();
  assertEquals(createdFiles, readBytes.keySet());
  for (Map.Entry<String, FixedBitSet> entry : readBytes.entrySet()) {
    final String file = entry.getKey();
    final FixedBitSet set = entry.getValue().clone();
    set.flip(0, set.length());
    final int next = set.nextSetBit(0);
    assertEquals("Byte at offset " + next + " of " + file + " was not read", DocIdSetIterator.NO_MORE_DOCS, next);
  }
  compoundDir.close();
  dir.close();
}
 

public void testConstantSlope() throws IOException {
  Directory dir = newDirectory();
  final int blockShift = TestUtil.nextInt(random(), DirectMonotonicWriter.MIN_BLOCK_SHIFT, DirectMonotonicWriter.MAX_BLOCK_SHIFT);
  final int numValues = TestUtil.nextInt(random(), 1, 1 << 20);
  final long min = random().nextLong();
  final long inc = random().nextInt(1 << random().nextInt(20));

  List<Long> actualValues = new ArrayList<>();
  for (int i = 0; i < numValues; ++i) {
    actualValues.add(min + inc * i);
  }

  final long dataLength;
  try (IndexOutput metaOut = dir.createOutput("meta", IOContext.DEFAULT);
      IndexOutput dataOut = dir.createOutput("data", IOContext.DEFAULT)) {
    DirectMonotonicWriter w = DirectMonotonicWriter.getInstance(metaOut, dataOut, numValues, blockShift);
    for (long v : actualValues) {
      w.add(v);
    }
    w.finish();
    dataLength = dataOut.getFilePointer();
  }

  try (IndexInput metaIn = dir.openInput("meta", IOContext.READONCE);
      IndexInput dataIn = dir.openInput("data", IOContext.DEFAULT)) {
    DirectMonotonicReader.Meta meta = DirectMonotonicReader.loadMeta(metaIn, numValues, blockShift);
    LongValues values = DirectMonotonicReader.getInstance(meta, dataIn.randomAccessSlice(0, dataLength));
    for (int i = 0; i < numValues; ++i) {
      assertEquals(actualValues.get(i).longValue(), values.get(i));
    }
    assertEquals(0, dataIn.getFilePointer());
  }

  dir.close();
}
 

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
      output.writeVInt(fi.getPointDimensionCount());
      if (fi.getPointDimensionCount() != 0) {
        output.writeVInt(fi.getPointIndexDimensionCount());
        output.writeVInt(fi.getPointNumBytes());
      }
    }
    CodecUtil.writeFooter(output);
  }
}
 

private void write(Directory directory) throws IOException {

    long nextGeneration = getNextPendingGeneration();
    String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
                                                                   "",
                                                                   nextGeneration);

    // Always advance the generation on write:
    generation = nextGeneration;
    
    IndexOutput segnOutput = null;
    boolean success = false;

    try {
      segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
      write(segnOutput);
      segnOutput.close();
      directory.sync(Collections.singleton(segmentFileName));
      success = true;
    } finally {
      if (success) {
        pendingCommit = true;
      } else {
        // We hit an exception above; try to close the file
        // but suppress any exception:
        IOUtils.closeWhileHandlingException(segnOutput);
        // Try not to leave a truncated segments_N file in
        // the index:
        IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName);
      }
    }
  }
 

public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
  this.directory = directory;
  this.segment = segment;
  boolean success = false;
  try {
    out = directory.createOutput(IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION), context);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(this);
    }
  }
}
 

private String writeFile(Directory dir, long length) throws IOException {
  IndexOutput output = dir.createOutput(OUT_DAT, IOContext.DEFAULT);
  for (long l = 0;l<length;l++) {
    output.writeByte((byte) 1);
  }
  output.close();
  return OUT_DAT;
}
 

public void testPackedInputOutput() throws IOException {
  final long[] longs = new long[random().nextInt(8192)];
  final int[] bitsPerValues = new int[longs.length];
  final boolean[] skip = new boolean[longs.length];
  for (int i = 0; i < longs.length; ++i) {
    final int bpv = RandomNumbers.randomIntBetween(random(), 1, 64);
    bitsPerValues[i] = random().nextBoolean() ? bpv : TestUtil.nextInt(random(), bpv, 64);
    if (bpv == 64) {
      longs[i] = random().nextLong();
    } else {
      longs[i] = TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
    }
    skip[i] = rarely();
  }

  final Directory dir = newDirectory();
  final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
  PackedDataOutput pout = new PackedDataOutput(out);
  long totalBits = 0;
  for (int i = 0; i < longs.length; ++i) {
    pout.writeLong(longs[i], bitsPerValues[i]);
    totalBits += bitsPerValues[i];
    if (skip[i]) {
      pout.flush();
      totalBits = 8 * (long) Math.ceil((double) totalBits / 8);
    }
  }
  pout.flush();
  assertEquals((long) Math.ceil((double) totalBits / 8), out.getFilePointer());
  out.close();
  final IndexInput in = dir.openInput("out.bin", IOContext.READONCE);
  final PackedDataInput pin = new PackedDataInput(in);
  for (int i = 0; i < longs.length; ++i) {
    assertEquals("" + i, longs[i], pin.readLong(bitsPerValues[i]));
    if (skip[i]) {
      pin.skipToNextByte();
    }
  }
  assertEquals((long) Math.ceil((double) totalBits / 8), in.getFilePointer());
  in.close();
  dir.close();
}
 

public void testEndPointer() throws IOException {
  final Directory dir = newDirectory();
  final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
  final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
  for (int i = 0; i < valueCount; ++i) {
    out.writeLong(0);
  }
  out.close();
  final IndexInput in = dir.openInput("tests.bin", newIOContext(random()));
  for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) {
    for (int bpv = 1; bpv <= 64; ++bpv) {
      for (PackedInts.Format format : PackedInts.Format.values()) {
        if (!format.isSupported(bpv)) {
          continue;
        }
        final long byteCount = format.byteCount(version, valueCount, bpv);
        String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;

        // test iterator
        in.seek(0L);
        final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1<<16));
        for (int i = 0; i < valueCount; ++i) {
          it.next();
        }
        assertEquals(msg, byteCount, in.getFilePointer());

        // test direct reader
        in.seek(0L);
        final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv);
        directReader.get(valueCount - 1);
        assertEquals(msg, byteCount, in.getFilePointer());

        // test reader
        in.seek(0L);
        PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv);
        assertEquals(msg, byteCount, in.getFilePointer());
       }
    }
  }
  in.close();
  dir.close();
}
 

@Nightly
public void testBlockReaderOverflow() throws IOException {
  final long valueCount = TestUtil.nextLong(random(), 1L + Integer.MAX_VALUE, (long) Integer.MAX_VALUE * 2);
  final int blockSize = 1 << TestUtil.nextInt(random(), 20, 22);
  final Directory dir = newDirectory();
  final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
  final BlockPackedWriter writer = new BlockPackedWriter(out, blockSize);
  long value = random().nextInt() & 0xFFFFFFFFL;
  long valueOffset = TestUtil.nextLong(random(), 0, valueCount - 1);
  for (long i = 0; i < valueCount; ) {
    assertEquals(i, writer.ord());
    if ((i & (blockSize - 1)) == 0 && (i + blockSize < valueOffset || i > valueOffset && i + blockSize < valueCount)) {
      writer.addBlockOfZeros();
      i += blockSize;
    } else if (i == valueOffset) {
      writer.add(value);
      ++i;
    } else {
      writer.add(0);
      ++i;
    }
  }
  writer.finish();
  out.close();
  final IndexInput in = dir.openInput("out.bin", IOContext.DEFAULT);
  final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
  it.skip(valueOffset);
  assertEquals(value, it.next());
  in.seek(0L);
  final BlockPackedReader reader = new BlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
  assertEquals(value, reader.get(valueOffset));
  for (int i = 0; i < 5; ++i) {
    final long offset = TestUtil.nextLong(random(), 0, valueCount - 1);
    if (offset == valueOffset) {
      assertEquals(value, reader.get(offset));
    } else {
      assertEquals(0, reader.get(offset));
    }
  }
  in.close();
  dir.close();
}