Java Code Examples for org.apache.lucene.store.IndexInput#close()

/** 
 * This test creates compound file based on a single file.
 * Files of different sizes are tested: 0, 1, 10, 100 bytes.
 */
public void testSingleFile() throws IOException {
  int data[] = new int[] { 0, 1, 10, 100 };
  for (int i=0; i<data.length; i++) {
    String testfile = "_" + i + ".test";
    Directory dir = newDirectory();
    SegmentInfo si = newSegmentInfo(dir, "_" + i);
    createSequenceFile(dir, testfile, (byte) 0, data[i], si.getId(), "suffix");
    
    si.setFiles(Collections.singleton(testfile));
    si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
    Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
    
    IndexInput expected = dir.openInput(testfile, newIOContext(random()));
    IndexInput actual = cfs.openInput(testfile, newIOContext(random()));
    assertSameStreams(testfile, expected, actual);
    assertSameSeekBehavior(testfile, expected, actual);
    expected.close();
    actual.close();
    cfs.close();
    dir.close();
  }
}
 

/** 
 * This test creates compound file based on two files.
 */
public void testTwoFiles() throws IOException {
  String files[] = { "_123.d1", "_123.d2" };
  Directory dir = newDirectory();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  createSequenceFile(dir, files[0], (byte) 0, 15, si.getId(), "suffix");
  createSequenceFile(dir, files[1], (byte) 0, 114, si.getId(), "suffix");
  
  si.setFiles(Arrays.asList(files));
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);

  for (String file : files) {
    IndexInput expected = dir.openInput(file, newIOContext(random()));
    IndexInput actual = cfs.openInput(file, newIOContext(random()));
    assertSameStreams(file, expected, actual);
    assertSameSeekBehavior(file, expected, actual);
    expected.close();
    actual.close();
  }

  cfs.close();
  dir.close();
}
 

public void testClonedStreamsClosing() throws IOException {
  Directory dir = newDirectory();
  Directory cr = createLargeCFS(dir);
  
  // basic clone
  IndexInput expected = dir.openInput("_123.f11", newIOContext(random()));
  
  IndexInput one = cr.openInput("_123.f11", newIOContext(random()));
  
  IndexInput two = one.clone();
  
  assertSameStreams("basic clone one", expected, one);
  expected.seek(0);
  assertSameStreams("basic clone two", expected, two);
  
  // Now close the compound reader
  cr.close();
  expected.close();
  dir.close();
}
 

@Test
public void test1() throws IOException {
  Random random = new Random(seed);
  RAMDirectory directory = new RAMDirectory();

  Cache cache = CacheIndexInputTest.getCache();
  CacheIndexOutput indexOutput = new CacheIndexOutput(null, "test", cache, directory, IOContext.DEFAULT);
  indexOutput.writeByte((byte) 1);
  indexOutput.writeByte((byte) 2);
  byte[] b = new byte[16000];
  random.nextBytes(b);
  indexOutput.writeBytes(b, 16000);
  indexOutput.close();

  IndexInput input = directory.openInput("test", IOContext.DEFAULT);
  assertEquals(16002, input.length());
  assertEquals(1, input.readByte());
  assertEquals(2, input.readByte());

  byte[] buf = new byte[16000];
  input.readBytes(buf, 0, 16000);
  input.close();
  assertArrayEquals(b, buf);
  directory.close();
}
 

public void testSingleValue() throws Exception {
  for (int bitsPerValue = 1; bitsPerValue <= 64; ++bitsPerValue) {
    Directory dir = newDirectory();
    IndexOutput out = dir.createOutput("out", newIOContext(random()));
    PackedInts.Writer w = PackedInts.getWriter(out, 1, bitsPerValue, PackedInts.DEFAULT);
    long value = 17L & PackedInts.maxValue(bitsPerValue);
    w.add(value);
    w.finish();
    final long end = out.getFilePointer();
    out.close();

    IndexInput in = dir.openInput("out", newIOContext(random()));
    Reader reader = PackedInts.getReader(in);
    String msg = "Impl=" + w.getClass().getSimpleName() + ", bitsPerValue=" + bitsPerValue;
    assertEquals(msg, 1, reader.size());
    assertEquals(msg, value, reader.get(0));
    assertEquals(msg, end, in.getFilePointer());
    in.close();

    dir.close();
  }
}
 

public void testRename() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();
  directory.rename("testing.test", "testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test"));
  assertTrue(slowFileExists(directory, "testing.test.renamed"));
  IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
  assertEquals(12345, input.readInt());
  assertEquals(input.getFilePointer(), input.length());
  input.close();
  directory.deleteFile("testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test.renamed"));
}
 

@Test
public void testMultiInstancesHdfsDirectoryTest1() throws IOException, InterruptedException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_root, "dir"));
  

  IndexOutput output = dir1.createOutput("a", IOContext.DEFAULT);
  output.writeInt(1234);
  output.close();
  
  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_root, "dir"));

  IndexInput input = dir2.openInput("a", IOContext.READ);
  assertEquals(4, input.length());
  assertEquals(1234, input.readInt());
  input.close();

  dir1.close();
  dir2.close();
}
 

private void doTestBpv(Directory directory, int bpv, long offset) throws Exception {
  MyRandom random = new MyRandom(random().nextLong());
  int numIters = TEST_NIGHTLY ? 100 : 10;
  for (int i = 0; i < numIters; i++) {
    long original[] = randomLongs(random, bpv);
    int bitsRequired = bpv == 64 ? 64 : DirectWriter.bitsRequired(1L<<(bpv-1));
    String name = "bpv" + bpv + "_" + i;
    IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
    for (long j = 0; j < offset; ++j) {
      output.writeByte((byte) random.nextInt());
    }
    DirectWriter writer = DirectWriter.getInstance(output, original.length, bitsRequired);
    for (int j = 0; j < original.length; j++) {
      writer.add(original[j]);
    }
    writer.finish();
    output.close();
    IndexInput input = directory.openInput(name, IOContext.DEFAULT);
    LongValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsRequired, offset);
    for (int j = 0; j < original.length; j++) {
      assertEquals("bpv=" + bpv, original[j], reader.get(j));
    }
    input.close();
  }
}
 

@Test
public void test2() throws IOException {
  Cache cache = getCache();
  RAMDirectory directory = new RAMDirectory();
  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  writeRandomData(size, random, output);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = new CacheIndexInput(null, name, input.clone(), cache);
  readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  readRandomDataShort(input, testInput, random, sampleSize);
  readRandomDataInt(input, testInput, random, sampleSize);
  readRandomDataLong(input, testInput, random, sampleSize);
  testInput.close();
  input.close();
  directory.close();
}
 

@Test
public void testLongReadAndClone() throws IOException {
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  String name = writeFile(dir,10*1000*1000);
  IndexInput input = dir.openInput(name, IOContext.DEFAULT);
  readFile(input,1000*1000);
  IndexInput clone = input.clone();
  clone.readByte();
  input.close();
}
 

@Ignore // memory hole
public void testOverflow() throws IOException {
  BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
  }
  final int blockBits = TestUtil.nextInt(random(), 14, 28);
  final int blockSize = 1 << blockBits;
  byte[] arr = new byte[TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
  for (int i = 0; i < arr.length; ++i) {
    arr[i] = (byte) i;
  }
  final long numBytes = (1L << 31) + TestUtil.nextInt(random(), 1, blockSize * 3);
  final PagedBytes p = new PagedBytes(blockBits);
  final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
  for (long i = 0; i < numBytes; ) {
    assertEquals(i, out.getFilePointer());
    final int len = (int) Math.min(arr.length, numBytes - i);
    out.writeBytes(arr, len);
    i += len;
  }
  assertEquals(numBytes, out.getFilePointer());
  out.close();
  final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
  p.copy(in, numBytes);
  final PagedBytes.Reader reader = p.freeze(random().nextBoolean());

  for (long offset : new long[] {0L, Integer.MAX_VALUE, numBytes - 1,
      TestUtil.nextLong(random(), 1, numBytes - 2)}) {
    BytesRef b = new BytesRef();
    reader.fillSlice(b, offset, 1);
    assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
  }
  in.close();
  dir.close();
}
 

private final Vector readDeleteableFiles() throws IOException {
  Vector result = new Vector();
  if (!directory.fileExists("deletable"))
    return result;

  IndexInput input = directory.openInput("deletable");
  try {
    for (int i = input.readInt(); i > 0; i--)	  // read file names
      result.addElement(input.readString());
  } finally {
    input.close();
  }
  return result;
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();

  IndexInput input = directory.openInput("testing.test", new IOContext());
  assertEquals(12345, input.readInt());
  input.close();

  listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", new IOContext());

  IndexInput input2 = input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(slowFileExists(directory, "testing.test.other"));
  assertTrue(slowFileExists(directory, "testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(slowFileExists(directory, "testing.test"));
}
 

public void testSimulatedCorruptIndex1() throws IOException {
    BaseDirectoryWrapper dir = newDirectory();
    dir.setCheckIndexOnClose(false); // we are corrupting it!

    IndexWriter writer = null;

    writer  = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));

    // add 100 documents
    for (int i = 0; i < 100; i++) {
        addDoc(writer);
    }

    // close
    writer.close();

    long gen = SegmentInfos.getLastCommitGeneration(dir);
    assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

    String fileNameIn = SegmentInfos.getLastCommitSegmentsFileName(dir);
    String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                               "",
                                                               1+gen);
    IndexInput in = dir.openInput(fileNameIn, newIOContext(random()));
    IndexOutput out = dir.createOutput(fileNameOut, newIOContext(random()));
    long length = in.length();
    for(int i=0;i<length-1;i++) {
      out.writeByte(in.readByte());
    }
    in.close();
    out.close();
    dir.deleteFile(fileNameIn);

    expectThrows(Exception.class, () -> {
      DirectoryReader.open(dir);
    });

    dir.close();
}
 

public void testSave() throws IOException {
  final int valueCount = TestUtil.nextInt(random(), 1, 2048);
  for (int bpv = 1; bpv <= 64; ++bpv) {
    final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
    final Directory directory = new ByteBuffersDirectory();
    List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
    for (PackedInts.Mutable mutable : packedInts) {
      for (int i = 0; i < mutable.size(); ++i) {
        mutable.set(i, random().nextInt(maxValue));
      }

      IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
      mutable.save(out);
      out.close();

      IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
      PackedInts.Reader reader = PackedInts.getReader(in);
      assertEquals(valueCount, reader.size());
      if (mutable instanceof Packed64SingleBlock) {
        // make sure that we used the right format so that the reader has
        // the same performance characteristics as the mutable that has been
        // serialized
        assertTrue(reader instanceof Packed64SingleBlock);
      } else {
        assertFalse(reader instanceof Packed64SingleBlock);
      }
      for (int i = 0; i < valueCount; ++i) {
        assertEquals(mutable.get(i), reader.get(i));
      }
      in.close();
      directory.deleteFile("packed-ints.bin");
    }
    directory.close();
  }
}
 

public void testSegmentHeaderLength() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeIndexHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
  output.writeString("this is the data");
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  input.seek(CodecUtil.indexHeaderLength("FooBar", "xyz"));
  assertEquals("this is the data", input.readString());
  input.close();
}
 

public void testChecksumEntireFile() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  CodecUtil.writeFooter(output);
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  CodecUtil.checksumEntireFile(input);
  input.close();
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {

  IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
  output.writeInt(12345);
  output.flush();
  output.close();

  IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
  assertEquals(12345, input.readInt());
  input.close();

  String[] listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);

  IndexInput input2 = (IndexInput) input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(directory.fileExists("testing.test.other"));
  assertTrue(directory.fileExists("testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(directory.fileExists("testing.test"));
}
 

public void testPackedInputOutput() throws IOException {
  final long[] longs = new long[random().nextInt(8192)];
  final int[] bitsPerValues = new int[longs.length];
  final boolean[] skip = new boolean[longs.length];
  for (int i = 0; i < longs.length; ++i) {
    final int bpv = RandomNumbers.randomIntBetween(random(), 1, 64);
    bitsPerValues[i] = random().nextBoolean() ? bpv : TestUtil.nextInt(random(), bpv, 64);
    if (bpv == 64) {
      longs[i] = random().nextLong();
    } else {
      longs[i] = TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
    }
    skip[i] = rarely();
  }

  final Directory dir = newDirectory();
  final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
  PackedDataOutput pout = new PackedDataOutput(out);
  long totalBits = 0;
  for (int i = 0; i < longs.length; ++i) {
    pout.writeLong(longs[i], bitsPerValues[i]);
    totalBits += bitsPerValues[i];
    if (skip[i]) {
      pout.flush();
      totalBits = 8 * (long) Math.ceil((double) totalBits / 8);
    }
  }
  pout.flush();
  assertEquals((long) Math.ceil((double) totalBits / 8), out.getFilePointer());
  out.close();
  final IndexInput in = dir.openInput("out.bin", IOContext.READONCE);
  final PackedDataInput pin = new PackedDataInput(in);
  for (int i = 0; i < longs.length; ++i) {
    assertEquals("" + i, longs[i], pin.readLong(bitsPerValues[i]));
    if (skip[i]) {
      pin.skipToNextByte();
    }
  }
  assertEquals((long) Math.ceil((double) totalBits / 8), in.getFilePointer());
  in.close();
  dir.close();
}
 

/** This test opens two files from a compound stream and verifies that
 *  their file positions are independent of each other.
 */
public void testRandomAccessClones() throws IOException {
  Directory dir = newDirectory();
  Directory cr = createLargeCFS(dir);
  
  // Open two files
  IndexInput e1 = cr.openInput("_123.f11", newIOContext(random()));
  IndexInput e2 = cr.openInput("_123.f3", newIOContext(random()));
  
  IndexInput a1 = e1.clone();
  IndexInput a2 = e2.clone();
  
  // Seek the first pair
  e1.seek(100);
  a1.seek(100);
  assertEquals(100, e1.getFilePointer());
  assertEquals(100, a1.getFilePointer());
  byte be1 = e1.readByte();
  byte ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now seek the second pair
  e2.seek(1027);
  a2.seek(1027);
  assertEquals(1027, e2.getFilePointer());
  assertEquals(1027, a2.getFilePointer());
  byte be2 = e2.readByte();
  byte ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Now make sure the first one didn't move
  assertEquals(101, e1.getFilePointer());
  assertEquals(101, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now more the first one again, past the buffer length
  e1.seek(1910);
  a1.seek(1910);
  assertEquals(1910, e1.getFilePointer());
  assertEquals(1910, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now make sure the second set didn't move
  assertEquals(1028, e2.getFilePointer());
  assertEquals(1028, a2.getFilePointer());
  be2 = e2.readByte();
  ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Move the second set back, again cross the buffer size
  e2.seek(17);
  a2.seek(17);
  assertEquals(17, e2.getFilePointer());
  assertEquals(17, a2.getFilePointer());
  be2 = e2.readByte();
  ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Finally, make sure the first set didn't move
  // Now make sure the first one didn't move
  assertEquals(1911, e1.getFilePointer());
  assertEquals(1911, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  e1.close();
  e2.close();
  a1.close();
  a2.close();
  cr.close();
  dir.close();
}