Java Code Examples for org.apache.lucene.store.Directory#createTempOutput()

The following examples show how to use org.apache.lucene.store.Directory#createTempOutput() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FieldsIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
FieldsIndexWriter(Directory dir, String name, String suffix, String extension,
    String codecName, byte[] id, int blockShift, IOContext ioContext) throws IOException {
  this.dir = dir;
  this.name = name;
  this.suffix = suffix;
  this.extension = extension;
  this.codecName = codecName;
  this.id = id;
  this.blockShift = blockShift;
  this.ioContext = ioContext;
  this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext);
  boolean success = false;
  try {
    CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);
    filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext);
    CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);
    success = true;
  } finally {
    if (success == false) {
      close();
    }
  }
}
 
Example 2
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testLeftoverTempFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  w.close();

  IndexOutput out = dir.createTempOutput("_0", "bkd", IOContext.DEFAULT);
  String tempName = out.getName();
  out.close();
  iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  w = new IndexWriter(dir, iwc);

  // Make sure IW deleted the unref'd file:
  try {
    dir.openInput(tempName, IOContext.DEFAULT);
    fail("did not hit exception");
  } catch (FileNotFoundException | NoSuchFileException e) {
    // expected
  }
  w.close();
  dir.close();
}
 
Example 3
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Check sorting data on an instance of {@link OfflineSorter}.
 */
private SortInfo checkSort(Directory dir, OfflineSorter sorter, byte[][] data) throws IOException {

  IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  writeAll(unsorted, data);

  IndexOutput golden = dir.createTempOutput("golden", "tmp", IOContext.DEFAULT);
  Arrays.sort(data, unsignedByteOrderComparator);
  writeAll(golden, data);

  String sorted = sorter.sort(unsorted.getName());
  //System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
  //System.out.println(sortInfo);
  assertFilesIdentical(dir, golden.getName(), sorted);

  return sorter.sortInfo;
}
 
Example 4
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(10*1024));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example 5
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Nightly
public void testFixedLengthHeap() throws Exception {
  // Make sure the RAM accounting is correct, i.e. if we are sorting fixed width
  // ints (4 bytes) then the heap used is really only 4 bytes per value:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    for (int i=0;i<1024*1024;i++) {
      random().nextBytes(bytes);
      w.write(bytes);
    }
    CodecUtil.writeFooter(out);
  }

  ExecutorService exec = randomExecutorServiceOrNull();
  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES, exec, TestUtil.nextInt(random(), 1, 4));
  sorter.sort(out.getName());
  if (exec != null) {
    exec.shutdownNow();
  }
  // 1 MB of ints with 4 MH heap allowed should have been sorted in a single heap partition:
  assertEquals(0, sorter.sortInfo.mergeRounds);
  dir.close();
}
 
Example 6
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testFixedLengthLiesLiesLies() throws Exception {
  // Make sure OfflineSorter catches me if I lie about the fixed value length:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    random().nextBytes(bytes);
    w.write(bytes);
    CodecUtil.writeFooter(out);
  }

  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Long.BYTES, null, 0);
  IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
    sorter.sort(out.getName());
    });
  assertEquals("value length is 4 but is supposed to always be 8", e.getMessage());
  dir.close();
}
 
Example 7
Source File: OfflinePointWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Create a new writer with an unknown number of incoming points */
public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength,
                          String desc, long expectedCount) throws IOException {
  this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT);
  this.name = out.getName();
  this.tempDir = tempDir;
  this.packedBytesLength = packedBytesLength;
  this.expectedCount = expectedCount;
}
 
Example 8
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example 9
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testOverNexting() throws Exception {
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    random().nextBytes(bytes);
    w.write(bytes);
    CodecUtil.writeFooter(out);
  }

  new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES, null, 0) {
    @Override
    protected ByteSequencesReader getReader(ChecksumIndexInput in, String name) throws IOException {
      ByteSequencesReader other = super.getReader(in, name);

      return new ByteSequencesReader(in, name) {

        private boolean alreadyEnded;
            
        @Override
        public BytesRef next() throws IOException {
          // if we returned null already, OfflineSorter should not call next() again
          assertFalse(alreadyEnded);
          BytesRef result = other.next();
          if (result == null) {
            alreadyEnded = true;
          }
          return result;
        }

        @Override
        public void close() throws IOException {
          other.close();
        }
      };
    }
  }.sort(out.getName());
  dir.close();
}
 
Example 10
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                // Replace length at the end with a too-long value:
                short v = in.readShort();
                assertEquals(256, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(5*1024));

    // This corruption made OfflineSorter fail with its own exception, but we verify and throw a CorruptIndexException
    // instead when checksums don't match.
    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example 11
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                tmpOut.copyBytes(in, 1025905);
                short v = in.readShort();
                assertEquals(254, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-1025905-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}