Java Code Examples for org.apache.lucene.util.IOUtils#deleteFilesIgnoringExceptions()

@Override
public BytesRefIterator iterator() throws IOException {
  if (sortedFileName == null) {
    closeWriter();
    
    boolean success = false;
    try {
      sortedFileName = sorter.sort(input.getName());
      success = true;
    } finally {
      if (success) {
        sorter.getDirectory().deleteFile(input.getName());
      } else {
        IOUtils.deleteFilesIgnoringExceptions(sorter.getDirectory(), input.getName());
      }
    }
    
    input = null;
  }
  
  return new ByteSequenceIterator(new OfflineSorter.ByteSequencesReader(sorter.getDirectory().openChecksumInput(sortedFileName, IOContext.READONCE), sortedFileName));
}
 

/**
 * Removes any written temporary files.
 */
@Override
public void close() throws IOException {
  try {
    closeWriter();
  } finally {
    IOUtils.deleteFilesIgnoringExceptions(sorter.getDirectory(),
                                          input == null ? null : input.getName(),
                                          sortedFileName);
  }
}
 

/**
 * Copies an existing {@code src} file from directory {@code from}
 * to a non-existent file {@code dest} in this directory.
 */
public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
  boolean success = false;
  try (IndexInput is = from.openInput(src, context);
       IndexOutput os = createOutput(dest, context)) {
    os.copyBytes(is, is.length());
    success = true;
  } finally {
    if (!success) {
      IOUtils.deleteFilesIgnoringExceptions(this, dest);
    }
  }
}
 

final void rollbackCommit(Directory dir) {
  if (pendingCommit) {
    pendingCommit = false;
    
    // we try to clean up our pending_segments_N

    // Must carefully compute fileName from "generation"
    // since lastGeneration isn't incremented:
    final String pending = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", generation);

    // Suppress so we keep throwing the original exception
    // in our caller
    IOUtils.deleteFilesIgnoringExceptions(dir, pending);
  }
}
 

private void write(Directory directory) throws IOException {

    long nextGeneration = getNextPendingGeneration();
    String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
                                                                   "",
                                                                   nextGeneration);

    // Always advance the generation on write:
    generation = nextGeneration;
    
    IndexOutput segnOutput = null;
    boolean success = false;

    try {
      segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
      write(segnOutput);
      segnOutput.close();
      directory.sync(Collections.singleton(segmentFileName));
      success = true;
    } finally {
      if (success) {
        pendingCommit = true;
      } else {
        // We hit an exception above; try to close the file
        // but suppress any exception:
        IOUtils.closeWhileHandlingException(segnOutput);
        // Try not to leave a truncated segments_N file in
        // the index:
        IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName);
      }
    }
  }
 

synchronized private void persist() throws IOException {
  String fileName = SNAPSHOTS_PREFIX + nextWriteGen;
  IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);   
    out.writeVInt(refCounts.size());
    for(Entry<Long,Integer> ent : refCounts.entrySet()) {
      out.writeVLong(ent.getKey());
      out.writeVInt(ent.getValue());
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
      IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
    } else {
      IOUtils.close(out);
    }
  }

  dir.sync(Collections.singletonList(fileName));
  
  if (nextWriteGen > 0) {
    String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
    // exception OK: likely it didn't exist
    IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
  }

  nextWriteGen++;
}
 

@Override
public void abort() {
  try {
    super.abort();
  } finally {
    if (tmpDirectory != null) {
      IOUtils.deleteFilesIgnoringExceptions(tmpDirectory,
          tmpDirectory.getTemporaryFiles().values());
    }
  }
}
 

@Override
void abort() {
  try {
    super.abort();
  } finally {
    if (tmpDirectory != null) {
      IOUtils.deleteFilesIgnoringExceptions(tmpDirectory,
          tmpDirectory.getTemporaryFiles().values());
    }
  }
}
 

private void close() throws IOException {
  try {
    IOUtils.close(reader);
  } finally {
    IOUtils.deleteFilesIgnoringExceptions(tempDir,
                                          tempInput == null ? null : tempInput.getName(),
                                          tempSortedFileName);
  }
}
 

/**
 * Download the given checksum URL to the destination and check the checksum
 * @param checksumURL URL for the checksum file
 * @param originalFile original file to calculate checksum of
 * @param checksumFile destination to download the checksum file to
 * @param hashFunc class used to calculate the checksum of the file
 * @return true if the checksum was validated, false if it did not exist
 * @throws Exception if the checksum failed to match
 */
public boolean downloadAndVerifyChecksum(URL checksumURL, Path originalFile, Path checksumFile,
                                         @Nullable DownloadProgress progress,
                                         TimeValue timeout, Checksummer hashFunc) throws Exception {
    try {
        if (download(checksumURL, checksumFile, progress, timeout)) {
            byte[] fileBytes = Files.readAllBytes(originalFile);
            List<String> checksumLines = Files.readAllLines(checksumFile, StandardCharsets.UTF_8);
            if (checksumLines.size() != 1) {
                throw new ElasticsearchCorruptionException("invalid format for checksum file (" +
                        hashFunc.name() + "), expected 1 line, got: " + checksumLines.size());
            }
            String checksumHex = checksumLines.get(0);
            String fileHex = hashFunc.checksum(fileBytes);
            if (fileHex.equals(checksumHex) == false) {
                throw new ElasticsearchCorruptionException("incorrect hash (" + hashFunc.name() +
                        "), file hash: [" + fileHex + "], expected: [" + checksumHex + "]");
            }
            return true;
        }
    } catch (FileNotFoundException | NoSuchFileException e) {
        // checksum file doesn't exist
        return false;
    } finally {
        IOUtils.deleteFilesIgnoringExceptions(checksumFile);
    }
    return false;
}
 

/**
 * Cleans up the index directory from old index files. This method uses the
 * last commit found by {@link #getLastCommit(Directory)}. If it matches the
 * expected segmentsFile, then all files not referenced by this commit point
 * are deleted.
 * <p>
 * <b>NOTE:</b> this method does a best effort attempt to clean the index
 * directory. It suppresses any exceptions that occur, as this can be retried
 * the next time.
 */
public static void cleanupOldIndexFiles(Directory dir, String segmentsFile, InfoStream infoStream) {
  try {
    IndexCommit commit = getLastCommit(dir);
    // commit == null means weird IO errors occurred, ignore them
    // if there were any IO errors reading the expected commit point (i.e.
    // segments files mismatch), then ignore that commit either.
    if (commit != null && commit.getSegmentsFileName().equals(segmentsFile)) {
      Set<String> commitFiles = new HashSet<>(commit.getFileNames());
      Matcher matcher = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
      for (String file : dir.listAll()) {
        if (!commitFiles.contains(file)
            && (matcher.reset(file).matches() || file.startsWith(IndexFileNames.SEGMENTS))) {
          // suppress exceptions, it's just a best effort
          IOUtils.deleteFilesIgnoringExceptions(dir, file);
        }
      }
    }
  } catch (Throwable t) {
    // ignore any errors that happen during this state and only log it. this
    // cleanup will have a chance to succeed the next time we get a new
    // revision.
    if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
      infoStream.message(INFO_STREAM_COMPONENT, "cleanupOldIndexFiles(): failed on error " + t.getMessage());
    }
  }
}
 

/**
 * Cleanup the index directory by deleting all given files. Called when file
 * copy or sync failed.
 */
public static void cleanupFilesOnFailure(Directory dir, List<String> files) {
  for (String file : files) {
    // suppress any exception because if we're here, it means copy
    // failed, and we must cleanup after ourselves.
    IOUtils.deleteFilesIgnoringExceptions(dir, file);
  }
}
 

public void cancel(String reason, Throwable exc) throws IOException {
  if (this.exc != null) {
    // Already cancelled
    return;
  }

  dest.message(String.format(Locale.ROOT, "top: cancel after copying %s; exc=%s:\n  files=%s\n  copiedFiles=%s",
                             Node.bytesToString(totBytesCopied),
                             exc,
                             files == null ? "null" : files.keySet(), copiedFiles.keySet()));

  if (exc == null) {
    exc = new Throwable();
  }

  this.exc = exc;
  this.cancelReason = reason;

  // Delete all temp files we wrote:
  IOUtils.deleteFilesIgnoringExceptions(dest.dir, copiedFiles.values());

  if (current != null) { 
    IOUtils.closeWhileHandlingException(current);
    if (Node.VERBOSE_FILES) {
      dest.message("remove partial file " + current.tmpName);
    }
    dest.deleter.deleteNewFile(current.tmpName);
    current = null;
  }
}
 

/** try to install our custom rule profile into sandbox_init() to block execution */
private static void macImpl(Path tmpFile) throws IOException {
    // first be defensive: we can give nice errors this way, at the very least.
    boolean supported = Constants.MAC_OS_X;
    if (supported == false) {
        throw new IllegalStateException("bug: should not be trying to initialize seatbelt for an unsupported OS");
    }

    // we couldn't link methods, could be some really ancient OS X (< Leopard) or some bug
    if (libc_mac == null) {
        throw new UnsupportedOperationException("seatbelt unavailable: could not link methods. requires Leopard or above.");
    }

    // write rules to a temporary file, which will be passed to sandbox_init()
    Path rules = Files.createTempFile(tmpFile, "es", "sb");
    Files.write(rules, Collections.singleton(SANDBOX_RULES), StandardCharsets.UTF_8);

    boolean success = false;
    try {
        PointerByReference errorRef = new PointerByReference();
        int ret = libc_mac.sandbox_init(rules.toAbsolutePath().toString(), SANDBOX_NAMED, errorRef);
        // if sandbox_init() fails, add the message from the OS (e.g. syntax error) and free the buffer
        if (ret != 0) {
            Pointer errorBuf = errorRef.getValue();
            RuntimeException e = new UnsupportedOperationException("sandbox_init(): " + errorBuf.getString(0));
            libc_mac.sandbox_free_error(errorBuf);
            throw e;
        }
        logger.debug("OS X seatbelt initialization successful");
        success = true;
    } finally {
        if (success) {
            Files.delete(rules);
        } else {
            IOUtils.deleteFilesIgnoringExceptions(rules);
        }
    }
}
 

/**
 * Closes streams, interrupts the download, may delete the
 * output file.
 */
void closeStreams() throws IOException {
    interrupt();
    if (success) {
        IOUtils.close(is, os);
    } else {
        IOUtils.closeWhileHandlingException(is, os);
        if (dest != null && Files.exists(dest)) {
            IOUtils.deleteFilesIgnoringExceptions(dest);
        }
    }
}
 

private synchronized void persist() throws IOException {
  String fileName = SNAPSHOTS_PREFIX + nextWriteGen;
  IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
    out.writeVInt(nameToDetailsMapping.size());
    for(Entry<String,SnapshotMetaData> ent : nameToDetailsMapping.entrySet()) {
      out.writeString(ent.getKey());
      out.writeString(ent.getValue().getIndexDirPath());
      out.writeVLong(ent.getValue().getGenerationNumber());
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
      IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
    } else {
      IOUtils.close(out);
    }
  }

  dir.sync(Collections.singletonList(fileName));

  if (nextWriteGen > 0) {
    String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
    // exception OK: likely it didn't exist
    IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
  }

  nextWriteGen++;
}
 

@Override
public void build(InputIterator iterator) throws IOException {
  if (iterator.hasPayloads()) {
    throw new IllegalArgumentException("this suggester doesn't support payloads");
  }
  if (iterator.hasContexts()) {
    throw new IllegalArgumentException("this suggester doesn't support contexts");
  }

  OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix);
  ExternalRefSorter externalSorter = new ExternalRefSorter(sorter);
  IndexOutput tempInput = tempDir.createTempOutput(tempFileNamePrefix, "input", IOContext.DEFAULT);
  String tempSortedFileName = null;

  OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
  OfflineSorter.ByteSequencesReader reader = null;

  // Push floats up front before sequences to sort them. For now, assume they are non-negative.
  // If negative floats are allowed some trickery needs to be done to find their byte order.
  count = 0;
  try {
    byte [] buffer = new byte [0];
    ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
    BytesRef spare;
    int inputLineCount = 0;
    while ((spare = iterator.next()) != null) {
      if (spare.length + 4 >= buffer.length) {
        buffer = ArrayUtil.grow(buffer, spare.length + 4);
      }

      output.reset(buffer);
      output.writeInt(encodeWeight(iterator.weight()));
      output.writeBytes(spare.bytes, spare.offset, spare.length);
      writer.write(buffer, 0, output.getPosition());
      inputLineCount++;
    }
    CodecUtil.writeFooter(tempInput);
    writer.close();

    // We don't know the distribution of scores and we need to bucket them, so we'll sort
    // and divide into equal buckets.
    tempSortedFileName = sorter.sort(tempInput.getName());
    tempDir.deleteFile(tempInput.getName());

    FSTCompletionBuilder builder = new FSTCompletionBuilder(
        buckets, externalSorter, sharedTailLength);

    reader = new OfflineSorter.ByteSequencesReader(tempDir.openChecksumInput(tempSortedFileName, IOContext.READONCE), tempSortedFileName);
    long line = 0;
    int previousBucket = 0;
    int previousScore = 0;
    ByteArrayDataInput input = new ByteArrayDataInput();
    BytesRef tmp2 = new BytesRef();
    while (true) {
      BytesRef scratch = reader.next();
      if (scratch == null) {
        break;
      }
      input.reset(scratch.bytes, scratch.offset, scratch.length);
      int currentScore = input.readInt();

      int bucket;
      if (line > 0 && currentScore == previousScore) {
        bucket = previousBucket;
      } else {
        bucket = (int) (line * buckets / inputLineCount);
      }
      previousScore = currentScore;
      previousBucket = bucket;

      // Only append the input, discard the weight.
      tmp2.bytes = scratch.bytes;
      tmp2.offset = scratch.offset + input.getPosition();
      tmp2.length = scratch.length - input.getPosition();
      builder.add(tmp2, bucket);

      line++;
      count++;
    }

    // The two FSTCompletions share the same automaton.
    this.higherWeightsCompletion = builder.build();
    this.normalCompletion = new FSTCompletion(
        higherWeightsCompletion.getFST(), false, exactMatchFirst);
    
  } finally {
    IOUtils.closeWhileHandlingException(reader, writer, externalSorter);
    IOUtils.deleteFilesIgnoringExceptions(tempDir, tempInput.getName(), tempSortedFileName);
  }
}
 

private boolean downloadFile() throws FileNotFoundException, IOException {
    IOException lastEx = null;
    for (int i = 0; i < 3; i++) {
        // this three attempt trick is to get round quirks in different
        // Java implementations. Some of them take a few goes to bind
        // property; we ignore the first couple of such failures.
        try {
            is = connection.getInputStream();
            break;
        } catch (IOException ex) {
            lastEx = ex;
        }
    }
    if (is == null) {
        throw lastEx;
    }

    os = Files.newOutputStream(dest);
    progress.beginDownload();
    boolean finished = false;
    try {
        byte[] buffer = new byte[1024 * 100];
        int length;
        while (!isInterrupted() && (length = is.read(buffer)) >= 0) {
            os.write(buffer, 0, length);
            progress.onTick();
        }
        finished = !isInterrupted();
    } finally {
        if (!finished) {
            // we have started to (over)write dest, but failed.
            // Try to delete the garbage we'd otherwise leave
            // behind.
            IOUtils.closeWhileHandlingException(os, is);
            IOUtils.deleteFilesIgnoringExceptions(dest);
        } else {
            IOUtils.close(os, is);
        }
    }
    progress.endDownload();
    return true;
}