Java Code Examples for org.apache.lucene.store.DataInput#readVInt()

/** Pulls CopyState off the wire */
static CopyState readCopyState(DataInput in) throws IOException {

  // Decode a new CopyState
  byte[] infosBytes = new byte[in.readVInt()];
  in.readBytes(infosBytes, 0, infosBytes.length);

  long gen = in.readVLong();
  long version = in.readVLong();
  Map<String,FileMetaData> files = readFilesMetaData(in);

  int count = in.readVInt();
  Set<String> completedMergeFiles = new HashSet<>();
  for(int i=0;i<count;i++) {
    completedMergeFiles.add(in.readString());
  }
  long primaryGen = in.readVLong();

  return new CopyState(files, version, gen, infosBytes, completedMergeFiles, primaryGen, null);
}
 

private void handleAddDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid") || name.equals("marker")) {
      doc.add(new StringField(name, value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }
  writer.addDocument(doc);
}
 

public BlockHeader read(DataInput input, BlockHeader reuse) throws IOException {
  int linesCount = input.readVInt();
  if (linesCount <= 0 || linesCount > UniformSplitTermsWriter.MAX_NUM_BLOCK_LINES) {
    throw new CorruptIndexException("Illegal number of lines in block: " + linesCount, input);
  }

  long baseDocsFP = input.readVLong();
  long basePositionsFP = input.readVLong();
  long basePayloadsFP = input.readVLong();

  int termStatesBaseOffset = input.readVInt();
  if (termStatesBaseOffset < 0) {
    throw new CorruptIndexException("Illegal termStatesBaseOffset= " + termStatesBaseOffset, input);
  }
  int middleTermOffset = input.readVInt();
  if (middleTermOffset < 0) {
    throw new CorruptIndexException("Illegal middleTermOffset= " + middleTermOffset, input);
  }

  BlockHeader blockHeader = reuse == null ? new BlockHeader() : reuse;
  return blockHeader.reset(linesCount, baseDocsFP, basePositionsFP, basePayloadsFP, termStatesBaseOffset, middleTermOffset);
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    in.skipBytes(bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos && (code & 1) == 0) {
      in.readVLong();
    }
  }
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  int len = in.readVInt();
  in.skipBytes(len);
  in.readVLong();
  in.readVLong();
}
 

@Override
public void decodeTerm(DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
  throws IOException {
  final IDVersionTermState termState = (IDVersionTermState) _termState;
  termState.docID = in.readVInt();
  if (absolute) {
    termState.idVersion = in.readVLong();
  } else {
    termState.idVersion += in.readZLong();
  }
}
 

@Override
public BytesRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final BytesRef output = new BytesRef(len);
    in.readBytes(output.bytes, 0, len);
    output.length = len;
    return output;
  }
}
 

/** Reads one BYTE1/2/4 label from the provided {@link DataInput}. */
public int readLabel(DataInput in) throws IOException {
  final int v;
  if (inputType == INPUT_TYPE.BYTE1) {
    // Unsigned byte:
    v = in.readByte() & 0xFF;
  } else if (inputType == INPUT_TYPE.BYTE2) {
    // Unsigned short:
    v = in.readShort() & 0xFFFF;
  } else { 
    v = in.readVInt();
  }
  return v;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  final int len = in.readVInt();
  for(int idx=0;idx<len;idx++) {
    in.readVInt();
  }
}
 

/**
 * Reads all the field ids in the current block line of the provided input.
 */
public int[] readFieldIds(DataInput termStatesInput, int numFields) throws IOException {
  int[] fieldIds = new int[numFields];
  for (int i = 0; i < numFields; i++) {
    fieldIds[i] = termStatesInput.readVInt();
  }
  return fieldIds;
}
 

static BytesRef readFrom(DataInput in, BytesRef scratch) throws IOException {
  scratch.length = in.readVInt();
  if (scratch.bytes.length < scratch.length) {
    scratch.bytes = ArrayUtil.grow(scratch.bytes, scratch.length);
  }
  in.readBytes(scratch.bytes, 0, scratch.length);
  return scratch;
}
 

/**
 * Reads the current line.
 *
 * @param isIncrementalEncodingSeed Whether the term is a seed of the
 *                                  incremental encoding. {@code true} for the first and
 *                                  middle term, {@code false} for other terms.
 * @param reuse                     A {@link BlockLine} instance to reuse; or null if none.
 */
public BlockLine readLine(DataInput blockInput, boolean isIncrementalEncodingSeed, BlockLine reuse) throws IOException {
  int termStateRelativeOffset = blockInput.readVInt();
  if (termStateRelativeOffset < 0) {
    throw new CorruptIndexException("Illegal termStateRelativeOffset= " + termStateRelativeOffset, blockInput);
  }
  return reuse == null ?
      new BlockLine(readIncrementallyEncodedTerm(blockInput, isIncrementalEncodingSeed, null), termStateRelativeOffset)
      : reuse.reset(readIncrementallyEncodedTerm(blockInput, isIncrementalEncodingSeed, reuse.termBytes), termStateRelativeOffset);
}
 

@Override
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
    maxAnalyzedPathsForOneInput = dataIn.readVInt();
    hasPayloads = dataIn.readByte() == 1;
  } finally {
    IOUtils.close(input);
  }
  return true;
}
 

@Override
public TermData read(DataInput in) throws IOException {
  byte[] bytes = null;
  int docFreq = 0;
  long totalTermFreq = -1;
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    bytes = new byte[bytesSize];
    in.readBytes(bytes, 0, bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos) {
      totalTermFreq = docFreq = code >>> 1;
      if ((code & 1) == 0) {
        totalTermFreq += in.readVLong();
      }
    } else {
      docFreq = code;
    }
  }
  return new TermData(bytes, docFreq, totalTermFreq);
}
 

@Override
public IntsRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final IntsRef output = new IntsRef(len);
    for(int idx=0;idx<len;idx++) {
      output.ints[idx] = in.readVInt();
    }
    output.length = len;
    return output;
  }
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len != 0) {
    in.skipBytes(len);
  }
}
 

@Override
public CharsRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final CharsRef output = new CharsRef(len);
    for(int idx=0;idx<len;idx++) {
      output.chars[idx] = (char) in.readVInt();
    }
    output.length = len;
    return output;
  }
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

/** Flushes all indexing ops to disk and notifies all replicas that they should now copy */
private void handleFlush(DataInput topIn, DataOutput topOut, BufferedOutputStream bos) throws IOException {
  Thread.currentThread().setName("flush");

  int atLeastMarkerCount = topIn.readVInt();

  int[] replicaTCPPorts;
  int[] replicaIDs;
  synchronized (this) {
    replicaTCPPorts = this.replicaTCPPorts;
    replicaIDs = this.replicaIDs;
  }

  message("now flush; " + replicaIDs.length + " replicas");

  if (flushAndRefresh()) {
    // Something did get flushed (there were indexing ops since the last flush):

    verifyAtLeastMarkerCount(atLeastMarkerCount, null);
 
   // Tell caller the version before pushing to replicas, so that even if we crash after this, caller will know what version we
    // (possibly) pushed to some replicas.  Alternatively we could make this 2 separate ops?
    long version = getCopyStateVersion();
    message("send flushed version=" + version);
    topOut.writeLong(version);
    bos.flush();

    // Notify current replicas:
    for(int i=0;i<replicaIDs.length;i++) {
      int replicaID = replicaIDs[i];
      try (Connection c = new Connection(replicaTCPPorts[i])) {
        message("send NEW_NRT_POINT to R" + replicaID + " at tcpPort=" + replicaTCPPorts[i]);
        c.out.writeByte(SimpleReplicaNode.CMD_NEW_NRT_POINT);
        c.out.writeVLong(version);
        c.out.writeVLong(primaryGen);
        c.out.writeInt(tcpPort);
        c.flush();
        // TODO: we should use multicast to broadcast files out to replicas
        // TODO: ... replicas could copy from one another instead of just primary
        // TODO: we could also prioritize one replica at a time?
      } catch (Throwable t) {
        message("top: failed to connect R" + replicaID + " for newNRTPoint; skipping: " + t.getMessage());
      }
    }
  } else {
    // No changes flushed:
    topOut.writeLong(-getCopyStateVersion());
  }
}