org.apache.lucene.store.DataInput Java Examples

The following examples show how to use org.apache.lucene.store.DataInput. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 22);
  shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
  for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
    long l0 = (tmp[tmpIdx+0] & MASK16_5) << 6;
    l0 |= (tmp[tmpIdx+1] & MASK16_5) << 1;
    l0 |= (tmp[tmpIdx+2] >>> 4) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+2] & MASK16_4) << 7;
    l1 |= (tmp[tmpIdx+3] & MASK16_5) << 2;
    l1 |= (tmp[tmpIdx+4] >>> 3) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+4] & MASK16_3) << 8;
    l2 |= (tmp[tmpIdx+5] & MASK16_5) << 3;
    l2 |= (tmp[tmpIdx+6] >>> 2) & MASK16_3;
    longs[longsIdx+2] = l2;
    long l3 = (tmp[tmpIdx+6] & MASK16_2) << 9;
    l3 |= (tmp[tmpIdx+7] & MASK16_5) << 4;
    l3 |= (tmp[tmpIdx+8] >>> 1) & MASK16_4;
    longs[longsIdx+3] = l3;
    long l4 = (tmp[tmpIdx+8] & MASK16_1) << 10;
    l4 |= (tmp[tmpIdx+9] & MASK16_5) << 5;
    l4 |= (tmp[tmpIdx+10] & MASK16_5) << 0;
    longs[longsIdx+4] = l4;
  }
}
 
Example #2
Source File: FuzzySet.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static FuzzySet deserialize(DataInput in) throws IOException
{
  int version=in.readInt();
  if (version == VERSION_SPI) {
    in.readString();
  }
  final HashFunction hashFunction = hashFunctionForVersion(version);
  int bloomSize=in.readInt();
  int numLongs=in.readInt();
  long[]longs=new long[numLongs];
  for (int i = 0; i < numLongs; i++) {
    longs[i]=in.readLong();
  }
  FixedBitSet bits = new FixedBitSet(longs,bloomSize+1);
  return new FuzzySet(bits,bloomSize,hashFunction);
}
 
Example #3
Source File: TestFSTDirectAddressing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void countFSTArcs(String fstFilePath) throws IOException {
  byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
  DataInput in = new ByteArrayDataInput(buf);
  FST<BytesRef> fst = new FST<>(in, in, ByteSequenceOutputs.getSingleton());
  BytesRefFSTEnum<BytesRef> fstEnum = new BytesRefFSTEnum<>(fst);
  int binarySearchArcCount = 0, directAddressingArcCount = 0, listArcCount = 0;
  while(fstEnum.next() != null) {
    if (fstEnum.arcs[fstEnum.upto].bytesPerArc() == 0) {
      listArcCount ++;
    } else if (fstEnum.arcs[fstEnum.upto].nodeFlags() == FST.ARCS_FOR_DIRECT_ADDRESSING) {
      directAddressingArcCount ++;
    } else {
      binarySearchArcCount ++;
    }
  }
  System.out.println("direct addressing arcs = " + directAddressingArcCount
      + ", binary search arcs = " + binarySearchArcCount
      + " list arcs = " + listArcCount);
}
 
Example #4
Source File: CompressingStoredFieldsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Reads a double in a variable-length format.  Reads between one and
 * nine bytes. Small integral values typically take fewer bytes.
 */
static double readZDouble(DataInput in) throws IOException {
  int b = in.readByte() & 0xFF;
  if (b == 0xFF) {
    // negative value
    return Double.longBitsToDouble(in.readLong());
  } else if (b == 0xFE) {
    // float
    return Float.intBitsToFloat(in.readInt());
  } else if ((b & 0x80) != 0) {
    // small integer [-1..124]
    return (b & 0x7f) - 1;
  } else {
    // positive double
    long bits = ((long) b) << 56 | ((in.readInt() & 0xFFFFFFFFL) << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL);
    return Double.longBitsToDouble(bits);
  }
}
 
Example #5
Source File: Packed64.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an array with content retrieved from the given DataInput.
 * @param in       a DataInput, positioned at the start of Packed64-content.
 * @param valueCount  the number of elements.
 * @param bitsPerValue the number of bits available for any given value.
 * @throws java.io.IOException if the values for the backing array could not
 *                             be retrieved.
 */
public Packed64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue)
                                                          throws IOException {
  super(valueCount, bitsPerValue);
  final PackedInts.Format format = PackedInts.Format.PACKED;
  final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read
  final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array
  blocks = new long[longCount];
  // read as many longs as we can
  for (int i = 0; i < byteCount / 8; ++i) {
    blocks[i] = in.readLong();
  }
  final int remaining = (int) (byteCount % 8);
  if (remaining != 0) {
    // read the last bytes
    long lastLong = 0;
    for (int i = 0; i < remaining; ++i) {
      lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8);
    }
    blocks[blocks.length - 1] = lastLong;
  }
  maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue);
  bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE;
}
 
Example #6
Source File: SimplePrimaryNode.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void handleAddDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid") || name.equals("marker")) {
      doc.add(new StringField(name, value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }
  writer.addDocument(doc);
}
 
Example #7
Source File: BytesStore.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Pulls bytes from the provided IndexInput.  */
public BytesStore(DataInput in, long numBytes, int maxBlockSize) throws IOException {
  int blockSize = 2;
  int blockBits = 1;
  while(blockSize < numBytes && blockSize < maxBlockSize) {
    blockSize *= 2;
    blockBits++;
  }
  this.blockBits = blockBits;
  this.blockSize = blockSize;
  this.blockMask = blockSize-1;
  long left = numBytes;
  while(left > 0) {
    final int chunk = (int) Math.min(blockSize, left);
    byte[] block = new byte[chunk];
    in.readBytes(block, 0, block.length);
    blocks.add(block);
    left -= chunk;
  }

  // So .getPosition still works
  nextWrite = blocks.get(blocks.size()-1).length;
}
 
Example #8
Source File: UpToTwoPositiveIntOutputs.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Object read(DataInput in) throws IOException {
  final long code = in.readVLong();
  if ((code & 1) == 0) {
    // single long
    final long v = code >>> 1;
    if (v == 0) {
      return NO_OUTPUT;
    } else {
      return Long.valueOf(v);
    }
  } else {
    // two longs
    final long first = code >>> 1;
    final long second = in.readVLong();
    return new TwoLongs(first, second);
  }
}
 
Example #9
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 40);
  shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
  for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK32_12) << 8;
    l0 |= (tmp[tmpIdx+1] >>> 4) & MASK32_8;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK32_4) << 16;
    l1 |= (tmp[tmpIdx+2] & MASK32_12) << 4;
    l1 |= (tmp[tmpIdx+3] >>> 8) & MASK32_4;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK32_8) << 12;
    l2 |= (tmp[tmpIdx+4] & MASK32_12) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example #10
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 10);
  shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
  for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK8_3) << 2;
    l0 |= (tmp[tmpIdx+1] >>> 1) & MASK8_2;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK8_1) << 4;
    l1 |= (tmp[tmpIdx+2] & MASK8_3) << 1;
    l1 |= (tmp[tmpIdx+3] >>> 2) & MASK8_1;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK8_2) << 3;
    l2 |= (tmp[tmpIdx+4] & MASK8_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example #11
Source File: BlockHeader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public BlockHeader read(DataInput input, BlockHeader reuse) throws IOException {
  int linesCount = input.readVInt();
  if (linesCount <= 0 || linesCount > UniformSplitTermsWriter.MAX_NUM_BLOCK_LINES) {
    throw new CorruptIndexException("Illegal number of lines in block: " + linesCount, input);
  }

  long baseDocsFP = input.readVLong();
  long basePositionsFP = input.readVLong();
  long basePayloadsFP = input.readVLong();

  int termStatesBaseOffset = input.readVInt();
  if (termStatesBaseOffset < 0) {
    throw new CorruptIndexException("Illegal termStatesBaseOffset= " + termStatesBaseOffset, input);
  }
  int middleTermOffset = input.readVInt();
  if (middleTermOffset < 0) {
    throw new CorruptIndexException("Illegal middleTermOffset= " + middleTermOffset, input);
  }

  BlockHeader blockHeader = reuse == null ? new BlockHeader() : reuse;
  return blockHeader.reset(linesCount, baseDocsFP, basePositionsFP, basePayloadsFP, termStatesBaseOffset, middleTermOffset);
}
 
Example #12
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 20);
  shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
  for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_6) << 4;
    l0 |= (tmp[tmpIdx+1] >>> 2) & MASK16_4;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK16_2) << 8;
    l1 |= (tmp[tmpIdx+2] & MASK16_6) << 2;
    l1 |= (tmp[tmpIdx+3] >>> 4) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK16_4) << 6;
    l2 |= (tmp[tmpIdx+4] & MASK16_6) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example #13
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 26);
  shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
  for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_3) << 10;
    l0 |= (tmp[tmpIdx+1] & MASK16_3) << 7;
    l0 |= (tmp[tmpIdx+2] & MASK16_3) << 4;
    l0 |= (tmp[tmpIdx+3] & MASK16_3) << 1;
    l0 |= (tmp[tmpIdx+4] >>> 2) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+4] & MASK16_2) << 11;
    l1 |= (tmp[tmpIdx+5] & MASK16_3) << 8;
    l1 |= (tmp[tmpIdx+6] & MASK16_3) << 5;
    l1 |= (tmp[tmpIdx+7] & MASK16_3) << 2;
    l1 |= (tmp[tmpIdx+8] >>> 1) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+8] & MASK16_1) << 12;
    l2 |= (tmp[tmpIdx+9] & MASK16_3) << 9;
    l2 |= (tmp[tmpIdx+10] & MASK16_3) << 6;
    l2 |= (tmp[tmpIdx+11] & MASK16_3) << 3;
    l2 |= (tmp[tmpIdx+12] & MASK16_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 
Example #14
Source File: SimpleTransLog.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void replayAddDocument(Connection c, NodeProcess primary, DataInput in) throws IOException {
  String id = in.readString();

  Document doc = new Document();
  doc.add(new StringField("docid", id, Field.Store.YES));

  String title = readNullableString(in);
  if (title != null) {
    doc.add(new StringField("title", title, Field.Store.NO));
    doc.add(new TextField("titleTokenized", title, Field.Store.NO));
  }
  String body = readNullableString(in);
  if (body != null) {
    doc.add(new TextField("body", body, Field.Store.NO));
  }
  String marker = readNullableString(in);
  if (marker != null) {
    //TestStressNRTReplication.message("xlog: replay marker=" + id);
    doc.add(new StringField("marker", marker, Field.Store.YES));
  }

  // For both add and update originally, we use updateDocument to replay,
  // because the doc could in fact already be in the index:
  // nocomit what if this fails?
  primary.addOrUpdateDocument(c, doc, false);
}
 
Example #15
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Restore a {@link ForUtil} from a {@link DataInput}.
 */
ForUtil(DataInput in) throws IOException {
  int packedIntsVersion = in.readVInt();
  PackedInts.checkVersion(packedIntsVersion);
  encodedSizes = new int[33];
  encoders = new PackedInts.Encoder[33];
  decoders = new PackedInts.Decoder[33];
  iterations = new int[33];

  for (int bpv = 1; bpv <= 32; ++bpv) {
    final int code = in.readVInt();
    final int formatId = code >>> 5;
    final int bitsPerValue = (code & 31) + 1;

    final PackedInts.Format format = PackedInts.Format.byId(formatId);
    assert format.isSupported(bitsPerValue);
    encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);
    encoders[bpv] = PackedInts.getEncoder(
        format, packedIntsVersion, bitsPerValue);
    decoders[bpv] = PackedInts.getDecoder(
        format, packedIntsVersion, bitsPerValue);
    iterations[bpv] = computeIterations(decoders[bpv]);
  }
}
 
Example #16
Source File: TokenInfoDictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  this.fst = new TokenInfoFST(fst);
}
 
Example #17
Source File: CharSequenceOutputs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void skipOutput(DataInput in) throws IOException {
  final int len = in.readVInt();
  for(int idx=0;idx<len;idx++) {
    in.readVInt();
  }
}
 
Example #18
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 2);
  shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);
  shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);
  shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);
  shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);
  shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);
  shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);
  shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);
  shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);
}
 
Example #19
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 12);
  shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);
  for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK8_2) << 4;
    l0 |= (tmp[tmpIdx+1] & MASK8_2) << 2;
    l0 |= (tmp[tmpIdx+2] & MASK8_2) << 0;
    longs[longsIdx+0] = l0;
  }
}
 
Example #20
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 6);
  shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);
  shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);
  for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {
    long l0 = (tmp[tmpIdx+0] & MASK8_2) << 1;
    l0 |= (tmp[tmpIdx+1] >>> 1) & MASK8_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK8_1) << 2;
    l1 |= (tmp[tmpIdx+2] & MASK8_2) << 0;
    longs[longsIdx+1] = l1;
  }
}
 
Example #21
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 4);
  shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);
  shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);
  shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);
  shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);
}
 
Example #22
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 14);
  shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);
  for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK8_1) << 6;
    l0 |= (tmp[tmpIdx+1] & MASK8_1) << 5;
    l0 |= (tmp[tmpIdx+2] & MASK8_1) << 4;
    l0 |= (tmp[tmpIdx+3] & MASK8_1) << 3;
    l0 |= (tmp[tmpIdx+4] & MASK8_1) << 2;
    l0 |= (tmp[tmpIdx+5] & MASK8_1) << 1;
    l0 |= (tmp[tmpIdx+6] & MASK8_1) << 0;
    longs[longsIdx+0] = l0;
  }
}
 
Example #23
Source File: SortField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected static Type readType(DataInput in) throws IOException {
  String type = in.readString();
  try {
    return Type.valueOf(type);
  }
  catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Can't deserialize SortField - unknown type " + type);
  }
}
 
Example #24
Source File: Rot13CypherTestUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static byte[] encode(DataInput bytesInput, int length) throws IOException {
  byte[] encodedBytes = new byte[length + ENCODING_OFFSET];
  for (int i = 0; i < length; i++) {
    encodedBytes[i + ENCODING_OFFSET] = (byte)(bytesInput.readByte() + ENCODING_ROTATION);
  }
  return encodedBytes;
}
 
Example #25
Source File: IntSequenceOutputs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public IntsRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final IntsRef output = new IntsRef(len);
    for(int idx=0;idx<len;idx++) {
      output.ints[idx] = in.readVInt();
    }
    output.length = len;
    return output;
  }
}
 
Example #26
Source File: CachedDecompressor.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Override
public void decompress(final DataInput in, final int originalLength, final int offset, final int length,
    final BytesRef bytes) throws IOException {
  if (in instanceof IndexInput) {
    IndexInput indexInput = (IndexInput) in;
    String name = indexInput.toString();
    long filePointer = indexInput.getFilePointer();

    Entry entry = _entry.get();
    if (!entry.isValid(indexInput, name, filePointer)) {
      entry.setup(indexInput, name, filePointer);
      entry._cache.grow(originalLength + 7);
      _decompressor.decompress(indexInput, originalLength, 0, originalLength, entry._cache);
      entry._cache.length = originalLength;
      entry._cache.offset = 0;
      _entry.set(entry);
    }
    if (bytes.bytes.length < originalLength + 7) {
      bytes.bytes = new byte[ArrayUtil.oversize(originalLength + 7, 1)];
    }
    System.arraycopy(entry._cache.bytes, entry._cache.offset, bytes.bytes, 0, length + offset);
    bytes.offset = offset;
    bytes.length = length;
  } else {
    _decompressor.decompress(in, originalLength, offset, length, bytes);
  }
}
 
Example #27
Source File: PackedInts.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Retrieve PackedInts as a {@link ReaderIterator}
 * @param in positioned at the beginning of a stored packed int structure.
 * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration)
 * @return an iterator to access the values
 * @throws IOException if the structure could not be retrieved.
 * @lucene.internal
 */
public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException {
  final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
  final int bitsPerValue = in.readVInt();
  assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
  final int valueCount = in.readVInt();
  final Format format = Format.byId(in.readVInt());
  return getReaderIteratorNoHeader(in, format, version, valueCount, bitsPerValue, mem);
}
 
Example #28
Source File: ForDeltaUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Decode deltas, compute the prefix sum and add {@code base} to all decoded longs.
 */
void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {
  final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
  if (bitsPerValue == 0) {
    prefixSumOfOnes(longs, base);
  } else {
    forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);
  }
}
 
Example #29
Source File: TSTLookup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  root = new TernaryTreeNode();
  readRecursively(input, root);
  return true;
}
 
Example #30
Source File: ForUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 48);
  shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);
  for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK32_8) << 16;
    l0 |= (tmp[tmpIdx+1] & MASK32_8) << 8;
    l0 |= (tmp[tmpIdx+2] & MASK32_8) << 0;
    longs[longsIdx+0] = l0;
  }
}