Java Code Examples for org.apache.lucene.store.IndexInput#readVInt()

The following examples show how to use org.apache.lucene.store.IndexInput#readVInt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CompressedIndexInput.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 
Example 2
Source File: CodecInfo.java    From mtas with Apache License 2.0 6 votes vote down vote up
/**
 * Instantiates a new index doc.
 *
 * @param ref
 *          the ref
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
public IndexDoc(Long ref) throws IOException {
  try {
    IndexInput inIndexDoc = indexInputList.get("doc");
    if (ref != null) {
      inIndexDoc.seek(ref);
    }
    docId = inIndexDoc.readVInt(); // docId
    fpIndexObjectId = inIndexDoc.readVLong(); // ref indexObjectId
    fpIndexObjectPosition = inIndexDoc.readVLong(); // ref
                                                    // indexObjectPosition
    fpIndexObjectParent = inIndexDoc.readVLong(); // ref indexObjectParent
    smallestObjectFilepointer = inIndexDoc.readVLong(); // offset
    objectRefApproxQuotient = inIndexDoc.readVInt(); // slope
    objectRefApproxOffset = inIndexDoc.readZLong(); // offset
    storageFlags = inIndexDoc.readByte(); // flag
    size = inIndexDoc.readVInt(); // number of objects
    minPosition = inIndexDoc.readVInt(); // minimum position
    maxPosition = inIndexDoc.readVInt(); // maximum position
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 
Example 3
Source File: BKDReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, BKDReaderDocIDSetIterator scratchIterator, int count, IntersectVisitor visitor) throws IOException {
  int i;
  for (i = 0; i < count;) {
    int length = in.readVInt();
    for(int dim = 0; dim < numDataDims; dim++) {
      int prefix = commonPrefixLengths[dim];
      in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
    }
    scratchIterator.reset(i, length);
    visitor.visit(scratchIterator, scratchPackedValue);
    i += length;
  }
  if (i != count) {
    throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);
  }
}
 
Example 4
Source File: NRTSuggester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
 * depending on the provided <code>fstLoadMode</code>
 */
public static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
  final FST<Pair<Long, BytesRef>> fst;
  if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
    OffHeapFSTStore store = new OffHeapFSTStore();
    IndexInput clone = input.clone();
    clone.seek(input.getFilePointer());
    fst = new FST<>(clone, clone, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()), store);
    input.seek(clone.getFilePointer() + store.size());
  } else {
    fst = new FST<>(input, input, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  }

  /* read some meta info */
  int maxAnalyzedPathsPerOutput = input.readVInt();
  /*
   * Label used to denote the end of an input in the FST and
   * the beginning of dedup bytes
   */
  int endByte = input.readVInt();
  int payloadSep = input.readVInt();
  return new NRTSuggester(fst, maxAnalyzedPathsPerOutput, payloadSep);
}
 
Example 5
Source File: RAMOnlyPostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FieldsProducer fieldsProducer(SegmentReadState readState)
  throws IOException {

  // Load our ID:
  final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION);
  IndexInput in = readState.directory.openInput(idFileName, readState.context);
  boolean success = false;
  final int id;
  try {
    CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST);
    id = in.readVInt();
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(in);
    } else {
      IOUtils.close(in);
    }
  }
  
  synchronized(state) {
    return state.get(id);
  }
}
 
Example 6
Source File: Lucene50PostingsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, int[] docBuffer,
    int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 
Example 7
Source File: Lucene84PostingsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, long[] docBuffer,
    long[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 
Example 8
Source File: Lucene50SkipReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 
Example 9
Source File: Lucene50ScoreSkipReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 
Example 10
Source File: DocIdsWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void readDeltaVInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException {
  int doc = 0;
  for (int i = 0; i < count; i++) {
    doc += in.readVInt();
    visitor.visit(doc);
  }
}
 
Example 11
Source File: DocIdsWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void readDeltaVInts(IndexInput in, int count, int[] docIDs) throws IOException {
  int doc = 0;
  for (int i = 0; i < count; i++) {
    doc += in.readVInt();
    docIDs[i] = doc;
  }
}
 
Example 12
Source File: Lucene84ScoreSkipReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 
Example 13
Source File: MonotonicBlockPackedReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private MonotonicBlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
  this.valueCount = valueCount;
  blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
  blockMask = blockSize - 1;
  final int numBlocks = numBlocks(valueCount, blockSize);
  minValues = new long[numBlocks];
  averages = new float[numBlocks];
  subReaders = new PackedInts.Reader[numBlocks];
  long sumBPV = 0;
  for (int i = 0; i < numBlocks; ++i) {
    minValues[i] = in.readZLong();
    averages[i] = Float.intBitsToFloat(in.readInt());
    final int bitsPerValue = in.readVInt();
    sumBPV += bitsPerValue;
    if (bitsPerValue > 64) {
      throw new IOException("Corrupted");
    }
    if (bitsPerValue == 0) {
      subReaders[i] = new PackedInts.NullReader(blockSize);
    } else {
      final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize);
      if (direct) {
        final long pointer = in.getFilePointer();
        subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
        in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));
      } else {
        subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
      }
    }
  }
  this.sumBPV = sumBPV;
}
 
Example 14
Source File: DiskDocValuesProducer.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
  NumericEntry entry = new NumericEntry();
  entry.packedIntsVersion = meta.readVInt();
  entry.offset = meta.readLong();
  entry.count = meta.readVLong();
  entry.blockSize = meta.readVInt();
  return entry;
}
 
Example 15
Source File: VersionBlockTreeTermsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 
Example 16
Source File: UniformSplitTermsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * @param indexInput {@link IndexInput} must be positioned to the fields metadata
 *                   details by calling {@link #seekFieldsMetadata(IndexInput)} before this call.
 * @param blockDecoder Optional block decoder, may be null if none.
 */
protected Collection<FieldMetadata> readFieldsMetadata(IndexInput indexInput, BlockDecoder blockDecoder, FieldInfos fieldInfos,
                                                              FieldMetadata.Serializer fieldMetadataReader, int maxNumDocs) throws IOException {
  int numFields = indexInput.readVInt();
  if (numFields < 0) {
    throw new CorruptIndexException("Illegal number of fields= " + numFields, indexInput);
  }
  return (blockDecoder != null && version >= VERSION_ENCODABLE_FIELDS_METADATA) ?
      readEncodedFieldsMetadata(numFields, indexInput, blockDecoder, fieldInfos, fieldMetadataReader, maxNumDocs)
      : readUnencodedFieldsMetadata(numFields, indexInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}
 
Example 17
Source File: DiskDocValuesProducer.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
  BinaryEntry entry = new BinaryEntry();
  entry.minLength = meta.readVInt();
  entry.maxLength = meta.readVInt();
  entry.count = meta.readVLong();
  entry.offset = meta.readLong();
  if (entry.minLength != entry.maxLength) {
    entry.addressesOffset = meta.readLong();
    entry.packedIntsVersion = meta.readVInt();
    entry.blockSize = meta.readVInt();
  }
  return entry;
}
 
Example 18
Source File: OrdsBlockTreeTermsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 
Example 19
Source File: VariableGapTermsIndexReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
  String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, 
                                                   state.segmentSuffix, 
                                                   VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
  final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context, true));
  boolean success = false;

  try {
    
    CodecUtil.checkIndexHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
                                     VariableGapTermsIndexWriter.VERSION_START,
                                     VariableGapTermsIndexWriter.VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
    
    CodecUtil.checksumEntireFile(in);

    seekDir(in);

    // Read directory
    final int numFields = in.readVInt();
    if (numFields < 0) {
      throw new CorruptIndexException("invalid numFields: " + numFields, in);
    }

    for(int i=0;i<numFields;i++) {
      final int field = in.readVInt();
      final long indexStart = in.readVLong();
      final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
      FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo, indexStart));
      if (previous != null) {
        throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }
}
 
Example 20
Source File: CodecSearchTree.java    From mtas with Apache License 2.0 4 votes vote down vote up
/**
 * Gets the mtas tree item.
 *
 * @param ref the ref
 * @param isSinglePoint the is single point
 * @param isStoreAdditionalIdAndRef the is store additional id and ref
 * @param nodeRefApproxOffset the node ref approx offset
 * @param in the in
 * @param objectRefApproxOffset the object ref approx offset
 * @return the mtas tree item
 * @throws IOException Signals that an I/O exception has occurred.
 */
private static MtasTreeItem getMtasTreeItem(Long ref,
    AtomicBoolean isSinglePoint, AtomicBoolean isStoreAdditionalIdAndRef,
    AtomicLong nodeRefApproxOffset, IndexInput in, long objectRefApproxOffset)
    throws IOException {
  try {
    Boolean isRoot = false;
    if (nodeRefApproxOffset.get() < 0) {
      isRoot = true;
    }
    in.seek(ref);
    if (isRoot) {
      nodeRefApproxOffset.set(in.readVLong());
      Byte flag = in.readByte();
      if ((flag
          & MtasTree.SINGLE_POSITION_TREE) == MtasTree.SINGLE_POSITION_TREE) {
        isSinglePoint.set(true);
      }
      if ((flag
          & MtasTree.STORE_ADDITIONAL_ID) == MtasTree.STORE_ADDITIONAL_ID) {
        isStoreAdditionalIdAndRef.set(true);
      }
    }
    int left = in.readVInt();
    int right = in.readVInt();
    int max = in.readVInt();
    Long leftChild = in.readVLong() + nodeRefApproxOffset.get();
    Long rightChild = in.readVLong() + nodeRefApproxOffset.get();
    int size = 1;
    if (!isSinglePoint.get()) {
      size = in.readVInt();
    }
    // initialize
    long[] objectRefs = new long[size];
    int[] objectAdditionalIds = null;
    long[] objectAdditionalRefs = null;
    // get first
    long objectRef = in.readVLong();
    long objectRefPrevious = objectRef + objectRefApproxOffset;
    objectRefs[0] = objectRefPrevious;
    if (isStoreAdditionalIdAndRef.get()) {
      objectAdditionalIds = new int[size];
      objectAdditionalRefs = new long[size];
      objectAdditionalIds[0] = in.readVInt();
      objectAdditionalRefs[0] = in.readVLong();
    }
    // get others
    for (int t = 1; t < size; t++) {
      objectRef = objectRefPrevious + in.readVLong();
      objectRefs[t] = objectRef;
      objectRefPrevious = objectRef;
      if (isStoreAdditionalIdAndRef.get()) {
        objectAdditionalIds[t] = in.readVInt();
        objectAdditionalRefs[t] = in.readVLong();
      }
    }
    return new MtasTreeItem(left, right, max, objectRefs, objectAdditionalIds,
        objectAdditionalRefs, ref, leftChild, rightChild);
  } catch (Exception e) {
    throw new IOException(e.getMessage());
  }
}