Java Code Examples for org.apache.lucene.store.Directory#createTempOutput()

FieldsIndexWriter(Directory dir, String name, String suffix, String extension,
    String codecName, byte[] id, int blockShift, IOContext ioContext) throws IOException {
  this.dir = dir;
  this.name = name;
  this.suffix = suffix;
  this.extension = extension;
  this.codecName = codecName;
  this.id = id;
  this.blockShift = blockShift;
  this.ioContext = ioContext;
  this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext);
  boolean success = false;
  try {
    CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);
    filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext);
    CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);
    success = true;
  } finally {
    if (success == false) {
      close();
    }
  }
}
 

public void testLeftoverTempFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  w.close();

  IndexOutput out = dir.createTempOutput("_0", "bkd", IOContext.DEFAULT);
  String tempName = out.getName();
  out.close();
  iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  w = new IndexWriter(dir, iwc);

  // Make sure IW deleted the unref'd file:
  try {
    dir.openInput(tempName, IOContext.DEFAULT);
    fail("did not hit exception");
  } catch (FileNotFoundException | NoSuchFileException e) {
    // expected
  }
  w.close();
  dir.close();
}
 

/**
 * Check sorting data on an instance of {@link OfflineSorter}.
 */
private SortInfo checkSort(Directory dir, OfflineSorter sorter, byte[][] data) throws IOException {

  IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  writeAll(unsorted, data);

  IndexOutput golden = dir.createTempOutput("golden", "tmp", IOContext.DEFAULT);
  Arrays.sort(data, unsignedByteOrderComparator);
  writeAll(golden, data);

  String sorted = sorter.sort(unsorted.getName());
  //System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
  //System.out.println(sortInfo);
  assertFilesIdentical(dir, golden.getName(), sorted);

  return sorter.sortInfo;
}
 

/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(10*1024));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

@Nightly
public void testFixedLengthHeap() throws Exception {
  // Make sure the RAM accounting is correct, i.e. if we are sorting fixed width
  // ints (4 bytes) then the heap used is really only 4 bytes per value:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    for (int i=0;i<1024*1024;i++) {
      random().nextBytes(bytes);
      w.write(bytes);
    }
    CodecUtil.writeFooter(out);
  }

  ExecutorService exec = randomExecutorServiceOrNull();
  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES, exec, TestUtil.nextInt(random(), 1, 4));
  sorter.sort(out.getName());
  if (exec != null) {
    exec.shutdownNow();
  }
  // 1 MB of ints with 4 MH heap allowed should have been sorted in a single heap partition:
  assertEquals(0, sorter.sortInfo.mergeRounds);
  dir.close();
}
 

public void testFixedLengthLiesLiesLies() throws Exception {
  // Make sure OfflineSorter catches me if I lie about the fixed value length:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    random().nextBytes(bytes);
    w.write(bytes);
    CodecUtil.writeFooter(out);
  }

  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Long.BYTES, null, 0);
  IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
    sorter.sort(out.getName());
    });
  assertEquals("value length is 4 but is supposed to always be 8", e.getMessage());
  dir.close();
}
 

/** Create a new writer with an unknown number of incoming points */
public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength,
                          String desc, long expectedCount) throws IOException {
  this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT);
  this.name = out.getName();
  this.tempDir = tempDir;
  this.packedBytesLength = packedBytesLength;
  this.expectedCount = expectedCount;
}
 

/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

public void testOverNexting() throws Exception {
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    random().nextBytes(bytes);
    w.write(bytes);
    CodecUtil.writeFooter(out);
  }

  new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES, null, 0) {
    @Override
    protected ByteSequencesReader getReader(ChecksumIndexInput in, String name) throws IOException {
      ByteSequencesReader other = super.getReader(in, name);

      return new ByteSequencesReader(in, name) {

        private boolean alreadyEnded;
            
        @Override
        public BytesRef next() throws IOException {
          // if we returned null already, OfflineSorter should not call next() again
          assertFalse(alreadyEnded);
          BytesRef result = other.next();
          if (result == null) {
            alreadyEnded = true;
          }
          return result;
        }

        @Override
        public void close() throws IOException {
          other.close();
        }
      };
    }
  }.sort(out.getName());
  dir.close();
}
 

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                // Replace length at the end with a too-long value:
                short v = in.readShort();
                assertEquals(256, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(5*1024));

    // This corruption made OfflineSorter fail with its own exception, but we verify and throw a CorruptIndexException
    // instead when checksums don't match.
    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                tmpOut.copyBytes(in, 1025905);
                short v = in.readShort();
                assertEquals(254, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-1025905-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}