Java Code Examples for org.apache.lucene.index.IndexFormatTooOldException

The following examples show how to use org.apache.lucene.index.IndexFormatTooOldException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: ChecksumBlobStoreFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        byte[] bytes = ByteStreams.toByteArray(inputStream);
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 
Example 2
Source Project: Elasticsearch   Source File: MetaDataStateFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.
 */
public final T read(Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
             // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            indexInput.readLong(); // version currently unused
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice, contentSize))) {
                    return fromXContent(parser);
                }
            }
        } catch(CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 
Example 3
Source Project: lucene-solr   Source File: CodecUtil.java    License: Apache License 2.0 6 votes vote down vote up
/** Like {@link
 *  #checkHeader(DataInput,String,int,int)} except this
 *  version assumes the first int has already been read
 *  and validated from the input. */
public static int checkHeaderNoMagic(DataInput in, String codec, int minVersion, int maxVersion) throws IOException {
  final String actualCodec = in.readString();
  if (!actualCodec.equals(codec)) {
    throw new CorruptIndexException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec, in);
  }

  final int actualVersion = in.readInt();
  if (actualVersion < minVersion) {
    throw new IndexFormatTooOldException(in, actualVersion, minVersion, maxVersion);
  }
  if (actualVersion > maxVersion) {
    throw new IndexFormatTooNewException(in, actualVersion, minVersion, maxVersion);
  }

  return actualVersion;
}
 
Example 4
Source Project: crate   Source File: ChecksumBlobStoreFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    final BytesReference bytes = Streams.readFully(blobContainer.readBlob(blobName));
    final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
    try (ByteArrayIndexInput indexInput =
             new ByteArrayIndexInput(resourceDesc, BytesReference.toBytes(bytes))) {
        CodecUtil.checksumEntireFile(indexInput);
        CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
        long filePointer = indexInput.getFilePointer();
        long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
        try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
                                                                 bytes.slice((int) filePointer, (int) contentSize), XContentType.SMILE)) {
            return reader.apply(parser);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
        // we trick this into a dedicated exception with the original stacktrace
        throw new CorruptStateException(ex);
    }
}
 
Example 5
private void failStoreIfCorrupted(Throwable t) {
    if (t instanceof CorruptIndexException || t instanceof IndexFormatTooOldException || t instanceof IndexFormatTooNewException) {
        try {
            store.markStoreCorrupted((IOException) t);
        } catch (IOException e) {
            logger.warn("store cannot be marked as corrupted", e);
        }
    }
}
 
Example 6
Source Project: crate   Source File: FileRestoreContext.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Restores a file
 *
 * @param fileInfo file to be restored
 */
private void restoreFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo, final Store store) throws IOException {
    boolean success = false;

    try (InputStream stream = fileInputStream(fileInfo)) {
        try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
            final byte[] buffer = new byte[bufferSize];
            int length;
            while ((length = stream.read(buffer)) > 0) {
                indexOutput.writeBytes(buffer, 0, length);
                recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.physicalName(), length);
            }
            Store.verify(indexOutput);
            indexOutput.close();
            store.directory().sync(Collections.singleton(fileInfo.physicalName()));
            success = true;
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            try {
                store.markStoreCorrupted(ex);
            } catch (IOException e) {
                LOGGER.warn("store cannot be marked as corrupted", e);
            }
            throw ex;
        } finally {
            if (success == false) {
                store.deleteQuiet(fileInfo.physicalName());
            }
        }
    }
}
 
Example 7
Source Project: crate   Source File: Store.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the last committed segments info for this store
 *
 * @throws IOException if the index is corrupted or the segments file is not present
 */
public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
    failIfCorrupted();
    try {
        return readSegmentsInfo(null, directory());
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
        markStoreCorrupted(ex);
        throw ex;
    }
}
 
Example 8
Source Project: crate   Source File: MetaDataStateFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.
 */
public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            if (xContentType != FORMAT) {
                throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType);
            }
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (InputStreamIndexInput in = new InputStreamIndexInput(slice, contentSize)) {
                    try (XContentParser parser = XContentFactory.xContent(FORMAT)
                            .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
                                in)) {
                        return fromXContent(parser);
                    }
                }
            }
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 
Example 9
Source Project: Elasticsearch   Source File: ExceptionsHelper.java    License: Apache License 2.0 4 votes vote down vote up
public static IOException unwrapCorruption(Throwable t) {
    return (IOException) unwrap(t, CorruptIndexException.class, 
                                   IndexFormatTooOldException.class, 
                                   IndexFormatTooNewException.class);
}
 
Example 10
/**
 * Restores a file
 * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
 * added to the {@code failures} list
 *
 * @param fileInfo file to be restored
 */
private void restoreFile(final FileInfo fileInfo) throws IOException {
    boolean success = false;

    try (InputStream partSliceStream = new PartSliceStream(blobContainer, fileInfo)) {
        final InputStream stream;
        if (restoreRateLimiter == null) {
            stream = partSliceStream;
        } else {
            stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreThrottleListener);
        }
        try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
            final byte[] buffer = new byte[BUFFER_SIZE];
            int length;
            while ((length = stream.read(buffer)) > 0) {
                indexOutput.writeBytes(buffer, 0, length);
                recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.name(), length);
            }
            Store.verify(indexOutput);
            indexOutput.close();
            // write the checksum
            if (fileInfo.metadata().hasLegacyChecksum()) {
                Store.LegacyChecksums legacyChecksums = new Store.LegacyChecksums();
                legacyChecksums.add(fileInfo.metadata());
                legacyChecksums.write(store);

            }
            store.directory().sync(Collections.singleton(fileInfo.physicalName()));
            success = true;
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            try {
                store.markStoreCorrupted(ex);
            } catch (IOException e) {
                logger.warn("store cannot be marked as corrupted", e);
            }
            throw ex;
        } finally {
            if (success == false) {
                store.deleteQuiet(fileInfo.physicalName());
            }
        }
    }
}
 
Example 11
Source Project: mtas   Source File: MtasFieldsProducer.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new mtas fields producer.
 *
 * @param state the state
 * @param name the name
 * @throws IOException Signals that an I/O exception has occurred.
 */
public MtasFieldsProducer(SegmentReadState state, String name)
    throws IOException {
  String postingsFormatName = null;
  indexInputList = new HashMap<>();
  indexInputOffsetList = new HashMap<>();
  version = MtasCodecPostingsFormat.VERSION_CURRENT;    
  postingsFormatName = addIndexInputToList("object", openMtasFile(state, name,
      MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION), postingsFormatName);
  addIndexInputToList("term",
      openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION),
      postingsFormatName);
  addIndexInputToList("prefix", openMtasFile(state, name,
      MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION), postingsFormatName);
  addIndexInputToList("field",
      openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION),
      postingsFormatName);
  addIndexInputToList("indexDocId",
      openMtasFile(state, name,
          MtasCodecPostingsFormat.MTAS_INDEX_DOC_ID_EXTENSION),
      postingsFormatName);
  addIndexInputToList("indexObjectId",
      openMtasFile(state, name,
          MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION),
      postingsFormatName);
  try {
    addIndexInputToList(
        "doc", openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
        postingsFormatName);
    addIndexInputToList("indexObjectPosition",
        openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION,
            version, version),
        postingsFormatName);
    addIndexInputToList("indexObjectParent",
        openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION,
            version, version),
        postingsFormatName);
  } catch (IndexFormatTooOldException e) {
    log.debug(e);
    throw new IOException(
        "This MTAS doesn't support your index version, please upgrade");
  }
  // Load the delegate postingsFormatName from this file
  this.delegateFieldsProducer = PostingsFormat.forName(postingsFormatName)
      .fieldsProducer(state);      
}
 
Example 12
Source Project: crate   Source File: ExceptionsHelper.java    License: Apache License 2.0 4 votes vote down vote up
public static IOException unwrapCorruption(Throwable t) {
    return (IOException) unwrap(t, CorruptIndexException.class,
                                   IndexFormatTooOldException.class,
                                   IndexFormatTooNewException.class);
}