Java Code Examples for org.apache.lucene.store.DataInput

The following examples show how to use org.apache.lucene.store.DataInput. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore a {@link ForUtil} from a {@link DataInput}.
 */
ForUtil(DataInput in) throws IOException {
  int packedIntsVersion = in.readVInt();
  PackedInts.checkVersion(packedIntsVersion);
  encodedSizes = new int[33];
  encoders = new PackedInts.Encoder[33];
  decoders = new PackedInts.Decoder[33];
  iterations = new int[33];

  for (int bpv = 1; bpv <= 32; ++bpv) {
    final int code = in.readVInt();
    final int formatId = code >>> 5;
    final int bitsPerValue = (code & 31) + 1;

    final PackedInts.Format format = PackedInts.Format.byId(formatId);
    assert format.isSupported(bitsPerValue);
    encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);
    encoders[bpv] = PackedInts.getEncoder(
        format, packedIntsVersion, bitsPerValue);
    decoders[bpv] = PackedInts.getDecoder(
        format, packedIntsVersion, bitsPerValue);
    iterations[bpv] = computeIterations(decoders[bpv]);
  }
}
 
Example 2
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 26);
  shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
  for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_3) << 10;
    l0 |= (tmp[tmpIdx+1] & MASK16_3) << 7;
    l0 |= (tmp[tmpIdx+2] & MASK16_3) << 4;
    l0 |= (tmp[tmpIdx+3] & MASK16_3) << 1;
    l0 |= (tmp[tmpIdx+4] >>> 2) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+4] & MASK16_2) << 11;
    l1 |= (tmp[tmpIdx+5] & MASK16_3) << 8;
    l1 |= (tmp[tmpIdx+6] & MASK16_3) << 5;
    l1 |= (tmp[tmpIdx+7] & MASK16_3) << 2;
    l1 |= (tmp[tmpIdx+8] >>> 1) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+8] & MASK16_1) << 12;
    l2 |= (tmp[tmpIdx+9] & MASK16_3) << 9;
    l2 |= (tmp[tmpIdx+10] & MASK16_3) << 6;
    l2 |= (tmp[tmpIdx+11] & MASK16_3) << 3;
    l2 |= (tmp[tmpIdx+12] & MASK16_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example 3
Source Project: lucene-solr   Source File: SimpleTransLog.java    License: Apache License 2.0 6 votes vote down vote up
private void replayAddDocument(Connection c, NodeProcess primary, DataInput in) throws IOException {
  String id = in.readString();

  Document doc = new Document();
  doc.add(new StringField("docid", id, Field.Store.YES));

  String title = readNullableString(in);
  if (title != null) {
    doc.add(new StringField("title", title, Field.Store.NO));
    doc.add(new TextField("titleTokenized", title, Field.Store.NO));
  }
  String body = readNullableString(in);
  if (body != null) {
    doc.add(new TextField("body", body, Field.Store.NO));
  }
  String marker = readNullableString(in);
  if (marker != null) {
    //TestStressNRTReplication.message("xlog: replay marker=" + id);
    doc.add(new StringField("marker", marker, Field.Store.YES));
  }

  // For both add and update originally, we use updateDocument to replay,
  // because the doc could in fact already be in the index:
  // nocomit what if this fails?
  primary.addOrUpdateDocument(c, doc, false);
}
 
Example 4
Source Project: lucene-solr   Source File: SimplePrimaryNode.java    License: Apache License 2.0 6 votes vote down vote up
private void handleAddDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid") || name.equals("marker")) {
      doc.add(new StringField(name, value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }
  writer.addDocument(doc);
}
 
Example 5
Source Project: lucene-solr   Source File: FuzzySet.java    License: Apache License 2.0 6 votes vote down vote up
public static FuzzySet deserialize(DataInput in) throws IOException
{
  int version=in.readInt();
  if (version == VERSION_SPI) {
    in.readString();
  }
  final HashFunction hashFunction = hashFunctionForVersion(version);
  int bloomSize=in.readInt();
  int numLongs=in.readInt();
  long[]longs=new long[numLongs];
  for (int i = 0; i < numLongs; i++) {
    longs[i]=in.readLong();
  }
  FixedBitSet bits = new FixedBitSet(longs,bloomSize+1);
  return new FuzzySet(bits,bloomSize,hashFunction);
}
 
Example 6
Source Project: lucene-solr   Source File: BytesStore.java    License: Apache License 2.0 6 votes vote down vote up
/** Pulls bytes from the provided IndexInput.  */
public BytesStore(DataInput in, long numBytes, int maxBlockSize) throws IOException {
  int blockSize = 2;
  int blockBits = 1;
  while(blockSize < numBytes && blockSize < maxBlockSize) {
    blockSize *= 2;
    blockBits++;
  }
  this.blockBits = blockBits;
  this.blockSize = blockSize;
  this.blockMask = blockSize-1;
  long left = numBytes;
  while(left > 0) {
    final int chunk = (int) Math.min(blockSize, left);
    byte[] block = new byte[chunk];
    in.readBytes(block, 0, block.length);
    blocks.add(block);
    left -= chunk;
  }

  // So .getPosition still works
  nextWrite = blocks.get(blocks.size()-1).length;
}
 
Example 7
Source Project: lucene-solr   Source File: BlockHeader.java    License: Apache License 2.0 6 votes vote down vote up
public BlockHeader read(DataInput input, BlockHeader reuse) throws IOException {
  int linesCount = input.readVInt();
  if (linesCount <= 0 || linesCount > UniformSplitTermsWriter.MAX_NUM_BLOCK_LINES) {
    throw new CorruptIndexException("Illegal number of lines in block: " + linesCount, input);
  }

  long baseDocsFP = input.readVLong();
  long basePositionsFP = input.readVLong();
  long basePayloadsFP = input.readVLong();

  int termStatesBaseOffset = input.readVInt();
  if (termStatesBaseOffset < 0) {
    throw new CorruptIndexException("Illegal termStatesBaseOffset= " + termStatesBaseOffset, input);
  }
  int middleTermOffset = input.readVInt();
  if (middleTermOffset < 0) {
    throw new CorruptIndexException("Illegal middleTermOffset= " + middleTermOffset, input);
  }

  BlockHeader blockHeader = reuse == null ? new BlockHeader() : reuse;
  return blockHeader.reset(linesCount, baseDocsFP, basePositionsFP, basePayloadsFP, termStatesBaseOffset, middleTermOffset);
}
 
Example 8
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 40);
  shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
  for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK32_12) << 8;
    l0 |= (tmp[tmpIdx+1] >>> 4) & MASK32_8;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK32_4) << 16;
    l1 |= (tmp[tmpIdx+2] & MASK32_12) << 4;
    l1 |= (tmp[tmpIdx+3] >>> 8) & MASK32_4;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK32_8) << 12;
    l2 |= (tmp[tmpIdx+4] & MASK32_12) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example 9
Source Project: lucene-solr   Source File: UpToTwoPositiveIntOutputs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object read(DataInput in) throws IOException {
  final long code = in.readVLong();
  if ((code & 1) == 0) {
    // single long
    final long v = code >>> 1;
    if (v == 0) {
      return NO_OUTPUT;
    } else {
      return Long.valueOf(v);
    }
  } else {
    // two longs
    final long first = code >>> 1;
    final long second = in.readVLong();
    return new TwoLongs(first, second);
  }
}
 
Example 10
Source Project: lucene-solr   Source File: Packed64.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an array with content retrieved from the given DataInput.
 * @param in       a DataInput, positioned at the start of Packed64-content.
 * @param valueCount  the number of elements.
 * @param bitsPerValue the number of bits available for any given value.
 * @throws java.io.IOException if the values for the backing array could not
 *                             be retrieved.
 */
public Packed64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue)
                                                          throws IOException {
  super(valueCount, bitsPerValue);
  final PackedInts.Format format = PackedInts.Format.PACKED;
  final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read
  final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array
  blocks = new long[longCount];
  // read as many longs as we can
  for (int i = 0; i < byteCount / 8; ++i) {
    blocks[i] = in.readLong();
  }
  final int remaining = (int) (byteCount % 8);
  if (remaining != 0) {
    // read the last bytes
    long lastLong = 0;
    for (int i = 0; i < remaining; ++i) {
      lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8);
    }
    blocks[blocks.length - 1] = lastLong;
  }
  maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue);
  bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE;
}
 
Example 11
Source Project: lucene-solr   Source File: TestFSTDirectAddressing.java    License: Apache License 2.0 6 votes vote down vote up
private static void countFSTArcs(String fstFilePath) throws IOException {
  byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
  DataInput in = new ByteArrayDataInput(buf);
  FST<BytesRef> fst = new FST<>(in, in, ByteSequenceOutputs.getSingleton());
  BytesRefFSTEnum<BytesRef> fstEnum = new BytesRefFSTEnum<>(fst);
  int binarySearchArcCount = 0, directAddressingArcCount = 0, listArcCount = 0;
  while(fstEnum.next() != null) {
    if (fstEnum.arcs[fstEnum.upto].bytesPerArc() == 0) {
      listArcCount ++;
    } else if (fstEnum.arcs[fstEnum.upto].nodeFlags() == FST.ARCS_FOR_DIRECT_ADDRESSING) {
      directAddressingArcCount ++;
    } else {
      binarySearchArcCount ++;
    }
  }
  System.out.println("direct addressing arcs = " + directAddressingArcCount
      + ", binary search arcs = " + binarySearchArcCount
      + " list arcs = " + listArcCount);
}
 
Example 12
Source Project: lucene-solr   Source File: CompressingStoredFieldsReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads a double in a variable-length format.  Reads between one and
 * nine bytes. Small integral values typically take fewer bytes.
 */
static double readZDouble(DataInput in) throws IOException {
  int b = in.readByte() & 0xFF;
  if (b == 0xFF) {
    // negative value
    return Double.longBitsToDouble(in.readLong());
  } else if (b == 0xFE) {
    // float
    return Float.intBitsToFloat(in.readInt());
  } else if ((b & 0x80) != 0) {
    // small integer [-1..124]
    return (b & 0x7f) - 1;
  } else {
    // positive double
    long bits = ((long) b) << 56 | ((in.readInt() & 0xFFFFFFFFL) << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL);
    return Double.longBitsToDouble(bits);
  }
}
 
Example 13
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 10);
  shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
  for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK8_3) << 2;
    l0 |= (tmp[tmpIdx+1] >>> 1) & MASK8_2;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK8_1) << 4;
    l1 |= (tmp[tmpIdx+2] & MASK8_3) << 1;
    l1 |= (tmp[tmpIdx+3] >>> 2) & MASK8_1;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK8_2) << 3;
    l2 |= (tmp[tmpIdx+4] & MASK8_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example 14
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 20);
  shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
  for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_6) << 4;
    l0 |= (tmp[tmpIdx+1] >>> 2) & MASK16_4;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK16_2) << 8;
    l1 |= (tmp[tmpIdx+2] & MASK16_6) << 2;
    l1 |= (tmp[tmpIdx+3] >>> 4) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK16_4) << 6;
    l2 |= (tmp[tmpIdx+4] & MASK16_6) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example 15
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 22);
  shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
  for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
    long l0 = (tmp[tmpIdx+0] & MASK16_5) << 6;
    l0 |= (tmp[tmpIdx+1] & MASK16_5) << 1;
    l0 |= (tmp[tmpIdx+2] >>> 4) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+2] & MASK16_4) << 7;
    l1 |= (tmp[tmpIdx+3] & MASK16_5) << 2;
    l1 |= (tmp[tmpIdx+4] >>> 3) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+4] & MASK16_3) << 8;
    l2 |= (tmp[tmpIdx+5] & MASK16_5) << 3;
    l2 |= (tmp[tmpIdx+6] >>> 2) & MASK16_3;
    longs[longsIdx+2] = l2;
    long l3 = (tmp[tmpIdx+6] & MASK16_2) << 9;
    l3 |= (tmp[tmpIdx+7] & MASK16_5) << 4;
    l3 |= (tmp[tmpIdx+8] >>> 1) & MASK16_4;
    longs[longsIdx+3] = l3;
    long l4 = (tmp[tmpIdx+8] & MASK16_1) << 10;
    l4 |= (tmp[tmpIdx+9] & MASK16_5) << 5;
    l4 |= (tmp[tmpIdx+10] & MASK16_5) << 0;
    longs[longsIdx+4] = l4;
  }
}
 
Example 16
Source Project: lucene-solr   Source File: PackedInts.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Retrieve PackedInts as a {@link ReaderIterator}
 * @param in positioned at the beginning of a stored packed int structure.
 * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration)
 * @return an iterator to access the values
 * @throws IOException if the structure could not be retrieved.
 * @lucene.internal
 */
public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException {
  final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
  final int bitsPerValue = in.readVInt();
  assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
  final int valueCount = in.readVInt();
  final Format format = Format.byId(in.readVInt());
  return getReaderIteratorNoHeader(in, format, version, valueCount, bitsPerValue, mem);
}
 
Example 17
Source Project: lucene-solr   Source File: BlockPackedReaderIterator.java    License: Apache License 2.0 5 votes vote down vote up
/** Reset the current reader to wrap a stream of <code>valueCount</code>
 * values contained in <code>in</code>. The block size remains unchanged. */
public void reset(DataInput in, long valueCount) {
  this.in = in;
  assert valueCount >= 0;
  this.valueCount = valueCount;
  off = blockSize;
  ord = 0;
}
 
Example 18
Source Project: crate   Source File: Checkpoint.java    License: Apache License 2.0 5 votes vote down vote up
static Checkpoint readCheckpointV5_0_0(final DataInput in) throws IOException {
    final long offset = in.readLong();
    final int numOps = in.readInt();
    final long generation = in.readLong();
    final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
    final long minTranslogGeneration = -1;
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 
Example 19
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 24);
  shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);
  for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK16_4) << 8;
    l0 |= (tmp[tmpIdx+1] & MASK16_4) << 4;
    l0 |= (tmp[tmpIdx+2] & MASK16_4) << 0;
    longs[longsIdx+0] = l0;
  }
}
 
Example 20
Source Project: lucene-solr   Source File: SimpleTransLog.java    License: Apache License 2.0 5 votes vote down vote up
private String readNullableString(DataInput in) throws IOException {
  byte b = in.readByte();
  if (b == 0) {
    return null;
  } else if (b == 1) {
    return in.readString();
  } else {
    throw new CorruptIndexException("invalid string lead byte " + b, in);
  }
}
 
Example 21
Source Project: lucene-solr   Source File: TestIndexInput.java    License: Apache License 2.0 5 votes vote down vote up
private void checkRandomReads(DataInput is) throws IOException {
  for (int i = 0; i < COUNT; i++) {
    assertEquals(INTS[i], is.readVInt());
    assertEquals(INTS[i], is.readInt());
    assertEquals(LONGS[i], is.readVLong());
    assertEquals(LONGS[i], is.readLong());
  }
}
 
Example 22
Source Project: lucene-solr   Source File: ByteSequenceOutputs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void skipOutput(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len != 0) {
    in.skipBytes(len);
  }
}
 
Example 23
Source Project: lucene-solr   Source File: SortedSetSortField.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortField readSortField(DataInput in) throws IOException {
  SortField sf = new SortedSetSortField(in.readString(), in.readInt() == 1, readSelectorType(in));
  int missingValue = in.readInt();
  if (missingValue == 1) {
    sf.setMissingValue(SortField.STRING_FIRST);
  }
  else if (missingValue == 2) {
    sf.setMissingValue(SortField.STRING_LAST);
  }
  return sf;
}
 
Example 24
Source Project: lucene-solr   Source File: SortField.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortField readSortField(DataInput in) throws IOException {
  SortField sf = new SortField(in.readString(), readType(in), in.readInt() == 1);
  if (in.readInt() == 1) {
    // missing object
    switch (sf.type) {
      case STRING:
        int missingString = in.readInt();
        if (missingString == 1) {
          sf.setMissingValue(STRING_FIRST);
        }
        else {
          sf.setMissingValue(STRING_LAST);
        }
        break;
      case INT:
        sf.setMissingValue(in.readInt());
        break;
      case LONG:
        sf.setMissingValue(in.readLong());
        break;
      case FLOAT:
        sf.setMissingValue(NumericUtils.sortableIntToFloat(in.readInt()));
        break;
      case DOUBLE:
        sf.setMissingValue(NumericUtils.sortableLongToDouble(in.readLong()));
        break;
      default:
        throw new IllegalArgumentException("Cannot deserialize sort of type " + sf.type);
    }
  }
  return sf;
}
 
Example 25
Source Project: lucene-solr   Source File: BlockPackedReaderIterator.java    License: Apache License 2.0 5 votes vote down vote up
static long readVLong(DataInput in) throws IOException {
  byte b = in.readByte();
  if (b >= 0) return b;
  long i = b & 0x7FL;
  b = in.readByte();
  i |= (b & 0x7FL) << 7;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 14;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 21;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 28;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 35;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 42;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 49;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0xFFL) << 56;
  return i;
}
 
Example 26
Source Project: lucene-solr   Source File: BinaryDocValuesWriter.java    License: Apache License 2.0 5 votes vote down vote up
BufferedBinaryDocValues(PackedLongValues lengths, int maxLength, DataInput bytesIterator, DocIdSetIterator docsWithFields) {
  this.value = new BytesRefBuilder();
  this.value.grow(maxLength);
  this.lengthsIterator = lengths.iterator();
  this.bytesIterator = bytesIterator;
  this.docsWithField = docsWithFields;
}
 
Example 27
Source Project: lucene-solr   Source File: DeltaBaseTermStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads a {@link BlockTermState} from the provided {@link DataInput}.
 * <p>
 * Simpler variant of {@link Lucene84PostingsReader#decodeTerm(DataInput, FieldInfo, BlockTermState, boolean)}.
 *
 * @param reuse {@link BlockTermState} to reuse; or null to create a new one.
 */
public BlockTermState readTermState(long baseDocStartFP, long basePosStartFP, long basePayStartFP,
                                    DataInput termStatesInput, FieldInfo fieldInfo, BlockTermState reuse) throws IOException {
  IndexOptions indexOptions = fieldInfo.getIndexOptions();
  boolean hasFreqs = indexOptions != IndexOptions.DOCS;
  boolean hasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;

  IntBlockTermState intTermState = reuse != null ? reset((IntBlockTermState) reuse) : new IntBlockTermState();

  intTermState.docFreq = termStatesInput.readVInt();
  intTermState.totalTermFreq = hasFreqs ?
      intTermState.docFreq + termStatesInput.readVLong() : intTermState.docFreq;
  assert intTermState.totalTermFreq >= intTermState.docFreq;

  if (intTermState.docFreq == 1) {
    intTermState.singletonDocID = termStatesInput.readVInt();
  } else {
    intTermState.docStartFP = baseDocStartFP + termStatesInput.readVLong();
  }

  if (hasPositions) {
    intTermState.posStartFP = basePosStartFP + termStatesInput.readVLong();
    boolean hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    if (hasOffsets || fieldInfo.hasPayloads()) {
      intTermState.payStartFP = basePayStartFP + termStatesInput.readVLong();
    }
    if (intTermState.totalTermFreq > BLOCK_SIZE) {
      intTermState.lastPosBlockOffset = termStatesInput.readVLong();
    }
  }
  if (intTermState.docFreq > BLOCK_SIZE) {
    intTermState.skipOffset = termStatesInput.readVLong();
  }
  return intTermState;
}
 
Example 28
Source Project: lucene-solr   Source File: PForUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Skip 128 integers.
 */
void skip(DataInput in) throws IOException {
  final int token = Byte.toUnsignedInt(in.readByte());
  final int bitsPerValue = token & 0x1f;
  final int numExceptions = token >>> 5;
  if (bitsPerValue == 0) {
    in.readVLong();
    in.skipBytes((numExceptions << 1));
  } else {
    in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));
  }
}
 
Example 29
Source Project: lucene-solr   Source File: BlockLine.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads {@code length} bytes from the given {@link DataInput} and stores
 * them at {@code offset} in {@code bytes.bytes}.
 */
protected void readBytes(DataInput input, BytesRef bytes, int offset, int length) throws IOException {
  assert bytes.offset == 0;
  bytes.length = offset + length;
  bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length);
  input.readBytes(bytes.bytes, offset, length);
}
 
Example 30
Source Project: lucene-solr   Source File: UniformSplitTermsReader.java    License: Apache License 2.0 5 votes vote down vote up
protected Collection<FieldMetadata> readEncodedFieldsMetadata(int numFields, DataInput metadataInput, BlockDecoder blockDecoder,
                                                              FieldInfos fieldInfos, FieldMetadata.Serializer fieldMetadataReader,
                                                              int maxNumDocs) throws IOException {
  long encodedLength = metadataInput.readVLong();
  if (encodedLength < 0) {
    throw new CorruptIndexException("Illegal encoded length: " + encodedLength, metadataInput);
  }
  BytesRef decodedBytes = blockDecoder.decode(metadataInput, encodedLength);
  DataInput decodedMetadataInput = new ByteArrayDataInput(decodedBytes.bytes, 0, decodedBytes.length);
  return readUnencodedFieldsMetadata(numFields, decodedMetadataInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}