Java Code Examples for org.apache.lucene.store.IndexInput#readVLong()

public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 

@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 

@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 

/**
 * Instantiates a new index doc.
 *
 * @param ref
 *          the ref
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
public IndexDoc(Long ref) throws IOException {
  try {
    IndexInput inIndexDoc = indexInputList.get("doc");
    if (ref != null) {
      inIndexDoc.seek(ref);
    }
    docId = inIndexDoc.readVInt(); // docId
    fpIndexObjectId = inIndexDoc.readVLong(); // ref indexObjectId
    fpIndexObjectPosition = inIndexDoc.readVLong(); // ref
                                                    // indexObjectPosition
    fpIndexObjectParent = inIndexDoc.readVLong(); // ref indexObjectParent
    smallestObjectFilepointer = inIndexDoc.readVLong(); // offset
    objectRefApproxQuotient = inIndexDoc.readVInt(); // slope
    objectRefApproxOffset = inIndexDoc.readZLong(); // offset
    storageFlags = inIndexDoc.readByte(); // flag
    size = inIndexDoc.readVInt(); // number of objects
    minPosition = inIndexDoc.readVInt(); // minimum position
    maxPosition = inIndexDoc.readVInt(); // maximum position
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
  final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);

  this.postingsReader = postingsReader;
  final IndexInput in = state.directory.openInput(termsFileName, state.context);

  boolean success = false;
  try {
    CodecUtil.checkIndexHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
                                     FSTTermsWriter.TERMS_VERSION_START,
                                     FSTTermsWriter.TERMS_VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
    CodecUtil.checksumEntireFile(in);
    this.postingsReader.init(in, state);
    seekDir(in);

    final FieldInfos fieldInfos = state.fieldInfos;
    final int numFields = in.readVInt();
    for (int i = 0; i < numFields; i++) {
      int fieldNumber = in.readVInt();
      FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
      long numTerms = in.readVLong();
      long sumTotalTermFreq = in.readVLong();
      // if frequencies are omitted, sumTotalTermFreq=sumDocFreq and we only write one value
      long sumDocFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? sumTotalTermFreq : in.readVLong();
      int docCount = in.readVInt();
      TermsReader current = new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount);
      TermsReader previous = fields.put(fieldInfo.name, current);
      checkFieldSummary(state.segmentInfo, in, current, previous);
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }
}
 

/**
 * Inits the.
 *
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
private void init() throws IOException {
  // move to begin
  IndexInput inField = indexInputList.get("field");
  inField.seek(indexInputOffsetList.get("field"));
  // store field references in memory
  fieldReferences = new HashMap<String, FieldReferences>();
  boolean doInit = true;
  while (doInit) {
    try {
      String field = inField.readString();
      long refIndexDoc = inField.readVLong();
      long refIndexDocId = inField.readVLong();
      int numberOfDocs = inField.readVInt();
      inField.readVLong(); // refTerm
      inField.readVInt(); // numberOfTerms
      long refPrefix = inField.readVLong();
      int numberOfPrefixes = inField.readVInt();
      fieldReferences.put(field, new FieldReferences(refIndexDoc,
          refIndexDocId, numberOfDocs, refPrefix, numberOfPrefixes));
    } catch (IOException e) {
      log.debug(e);
      doInit = false;
    }
  }
  // prefixReferences
  prefixReferences = new HashMap<String, LinkedHashMap<String, Long>>();
}
 

static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
  NumericEntry entry = new NumericEntry();
  entry.packedIntsVersion = meta.readVInt();
  entry.offset = meta.readLong();
  entry.count = meta.readVLong();
  entry.blockSize = meta.readVInt();
  return entry;
}
 

static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
  BinaryEntry entry = new BinaryEntry();
  entry.minLength = meta.readVInt();
  entry.maxLength = meta.readVInt();
  entry.count = meta.readVLong();
  entry.offset = meta.readLong();
  if (entry.minLength != entry.maxLength) {
    entry.addressesOffset = meta.readLong();
    entry.packedIntsVersion = meta.readVInt();
    entry.blockSize = meta.readVInt();
  }
  return entry;
}
 

public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
  String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, 
                                                   state.segmentSuffix, 
                                                   VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
  final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context, true));
  boolean success = false;

  try {
    
    CodecUtil.checkIndexHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
                                     VariableGapTermsIndexWriter.VERSION_START,
                                     VariableGapTermsIndexWriter.VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
    
    CodecUtil.checksumEntireFile(in);

    seekDir(in);

    // Read directory
    final int numFields = in.readVInt();
    if (numFields < 0) {
      throw new CorruptIndexException("invalid numFields: " + numFields, in);
    }

    for(int i=0;i<numFields;i++) {
      final int field = in.readVInt();
      final long indexStart = in.readVLong();
      final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
      FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo, indexStart));
      if (previous != null) {
        throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }
}
 

public FixedGapTermsIndexReader(SegmentReadState state) throws IOException {
  final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
  
  String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, 
                                                   state.segmentSuffix, 
                                                   FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
  final IndexInput in = state.directory.openInput(fileName, state.context);
  
  boolean success = false;

  try {
    
    CodecUtil.checkIndexHeader(in, FixedGapTermsIndexWriter.CODEC_NAME,
                                     FixedGapTermsIndexWriter.VERSION_CURRENT, 
                                     FixedGapTermsIndexWriter.VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
    
    CodecUtil.checksumEntireFile(in);
    
    indexInterval = in.readVInt();
    if (indexInterval < 1) {
      throw new CorruptIndexException("invalid indexInterval: " + indexInterval, in);
    }
    packedIntsVersion = in.readVInt();
    blocksize = in.readVInt();
    
    seekDir(in);

    // Read directory
    final int numFields = in.readVInt();     
    if (numFields < 0) {
      throw new CorruptIndexException("invalid numFields: " + numFields, in);
    }
    //System.out.println("FGR: init seg=" + segment + " div=" + indexDivisor + " nF=" + numFields);
    for(int i=0;i<numFields;i++) {
      final int field = in.readVInt();
      final long numIndexTerms = in.readVInt(); // TODO: change this to a vLong if we fix writer to support > 2B index terms
      if (numIndexTerms < 0) {
        throw new CorruptIndexException("invalid numIndexTerms: " + numIndexTerms, in);
      }
      final long termsStart = in.readVLong();
      final long indexStart = in.readVLong();
      final long packedIndexStart = in.readVLong();
      final long packedOffsetsStart = in.readVLong();
      if (packedIndexStart < indexStart) {
        throw new CorruptIndexException("invalid packedIndexStart: " + packedIndexStart + " indexStart: " + indexStart + "numIndexTerms: " + numIndexTerms, in);
      }
      final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
      FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, termBytes, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
      if (previous != null) {
        throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
    termBytesReader = termBytes.freeze(true);
  }
}
 

LegacyFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
  maxDoc = si.maxDoc();
  int[] docBases = new int[16];
  long[] startPointers = new long[16];
  int[] avgChunkDocs = new int[16];
  long[] avgChunkSizes = new long[16];
  PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];
  PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];

  final int packedIntsVersion = fieldsIndexIn.readVInt();

  int blockCount = 0;

  for (;;) {
    final int numChunks = fieldsIndexIn.readVInt();
    if (numChunks == 0) {
      break;
    }
    if (blockCount == docBases.length) {
      final int newSize = ArrayUtil.oversize(blockCount + 1, 8);
      docBases = ArrayUtil.growExact(docBases, newSize);
      startPointers = ArrayUtil.growExact(startPointers, newSize);
      avgChunkDocs = ArrayUtil.growExact(avgChunkDocs, newSize);
      avgChunkSizes = ArrayUtil.growExact(avgChunkSizes, newSize);
      docBasesDeltas = ArrayUtil.growExact(docBasesDeltas, newSize);
      startPointersDeltas = ArrayUtil.growExact(startPointersDeltas, newSize);
    }

    // doc bases
    docBases[blockCount] = fieldsIndexIn.readVInt();
    avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();
    final int bitsPerDocBase = fieldsIndexIn.readVInt();
    if (bitsPerDocBase > 32) {
      throw new CorruptIndexException("Corrupted bitsPerDocBase: " + bitsPerDocBase, fieldsIndexIn);
    }
    docBasesDeltas[blockCount] = PackedInts.getReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerDocBase);

    // start pointers
    startPointers[blockCount] = fieldsIndexIn.readVLong();
    avgChunkSizes[blockCount] = fieldsIndexIn.readVLong();
    final int bitsPerStartPointer = fieldsIndexIn.readVInt();
    if (bitsPerStartPointer > 64) {
      throw new CorruptIndexException("Corrupted bitsPerStartPointer: " + bitsPerStartPointer, fieldsIndexIn);
    }
    startPointersDeltas[blockCount] = PackedInts.getReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerStartPointer);

    ++blockCount;
  }

  this.docBases = ArrayUtil.copyOfSubArray(docBases, 0, blockCount);
  this.startPointers = ArrayUtil.copyOfSubArray(startPointers, 0, blockCount);
  this.avgChunkDocs = ArrayUtil.copyOfSubArray(avgChunkDocs, 0, blockCount);
  this.avgChunkSizes = ArrayUtil.copyOfSubArray(avgChunkSizes, 0, blockCount);
  this.docBasesDeltas = ArrayUtil.copyOfSubArray(docBasesDeltas, 0, blockCount);
  this.startPointersDeltas = ArrayUtil.copyOfSubArray(startPointersDeltas, 0, blockCount);
}
 

/**
 * Gets the mtas tree item.
 *
 * @param ref the ref
 * @param isSinglePoint the is single point
 * @param isStoreAdditionalIdAndRef the is store additional id and ref
 * @param nodeRefApproxOffset the node ref approx offset
 * @param in the in
 * @param objectRefApproxOffset the object ref approx offset
 * @return the mtas tree item
 * @throws IOException Signals that an I/O exception has occurred.
 */
private static MtasTreeItem getMtasTreeItem(Long ref,
    AtomicBoolean isSinglePoint, AtomicBoolean isStoreAdditionalIdAndRef,
    AtomicLong nodeRefApproxOffset, IndexInput in, long objectRefApproxOffset)
    throws IOException {
  try {
    Boolean isRoot = false;
    if (nodeRefApproxOffset.get() < 0) {
      isRoot = true;
    }
    in.seek(ref);
    if (isRoot) {
      nodeRefApproxOffset.set(in.readVLong());
      Byte flag = in.readByte();
      if ((flag
          & MtasTree.SINGLE_POSITION_TREE) == MtasTree.SINGLE_POSITION_TREE) {
        isSinglePoint.set(true);
      }
      if ((flag
          & MtasTree.STORE_ADDITIONAL_ID) == MtasTree.STORE_ADDITIONAL_ID) {
        isStoreAdditionalIdAndRef.set(true);
      }
    }
    int left = in.readVInt();
    int right = in.readVInt();
    int max = in.readVInt();
    Long leftChild = in.readVLong() + nodeRefApproxOffset.get();
    Long rightChild = in.readVLong() + nodeRefApproxOffset.get();
    int size = 1;
    if (!isSinglePoint.get()) {
      size = in.readVInt();
    }
    // initialize
    long[] objectRefs = new long[size];
    int[] objectAdditionalIds = null;
    long[] objectAdditionalRefs = null;
    // get first
    long objectRef = in.readVLong();
    long objectRefPrevious = objectRef + objectRefApproxOffset;
    objectRefs[0] = objectRefPrevious;
    if (isStoreAdditionalIdAndRef.get()) {
      objectAdditionalIds = new int[size];
      objectAdditionalRefs = new long[size];
      objectAdditionalIds[0] = in.readVInt();
      objectAdditionalRefs[0] = in.readVLong();
    }
    // get others
    for (int t = 1; t < size; t++) {
      objectRef = objectRefPrevious + in.readVLong();
      objectRefs[t] = objectRef;
      objectRefPrevious = objectRef;
      if (isStoreAdditionalIdAndRef.get()) {
        objectAdditionalIds[t] = in.readVInt();
        objectAdditionalRefs[t] = in.readVLong();
      }
    }
    return new MtasTreeItem(left, right, max, objectRefs, objectAdditionalIds,
        objectAdditionalRefs, ref, leftChild, rightChild);
  } catch (Exception e) {
    throw new IOException(e.getMessage());
  }
}