Java Code Examples for org.apache.lucene.store.ByteArrayDataInput#getPosition()

The following examples show how to use org.apache.lucene.store.ByteArrayDataInput#getPosition() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CompressingTermVectorsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
void reset(int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths,
    int[] payloadIndex, BytesRef payloads, ByteArrayDataInput in) {
  this.numTerms = numTerms;
  this.prefixLengths = prefixLengths;
  this.suffixLengths = suffixLengths;
  this.termFreqs = termFreqs;
  this.positionIndex = positionIndex;
  this.positions = positions;
  this.startOffsets = startOffsets;
  this.lengths = lengths;
  this.payloadIndex = payloadIndex;
  this.payloads = payloads;
  this.in = in;
  startPos = in.getPosition();
  reset();
}
Example 2
Source File: IDVersionSegmentTermsEnumFrame.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
public void setFloorData(ByteArrayDataInput in, BytesRef source) {
  final int numBytes = source.length - (in.getPosition() - source.offset);
  if (numBytes > floorData.length) {
    floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
  }
  System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes);
  floorDataReader.reset(floorData, 0, numBytes);
  numFollowFloorBlocks = floorDataReader.readVInt();
  nextFloorLabel = floorDataReader.readByte() & 0xff;
  //if (DEBUG) {
  //System.out.println("    setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
  //}
}
Example 3
Source File: SegmentTermsEnumFrame.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
public void setFloorData(ByteArrayDataInput in, BytesRef source) {
  final int numBytes = source.length - (in.getPosition() - source.offset);
  if (numBytes > floorData.length) {
    floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
  }
  System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes);
  floorDataReader.reset(floorData, 0, numBytes);
  numFollowFloorBlocks = floorDataReader.readVInt();
  nextFloorLabel = floorDataReader.readByte() & 0xff;
  //if (DEBUG) {
  //System.out.println("    setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
  //}
}
Example 4
Source File: BlockTreeTermsReader.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up 
public void setFloorData(ByteArrayDataInput in, BytesRef source) {
  final int numBytes = source.length - (in.getPosition() - source.offset);
  if (numBytes > floorData.length) {
    floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
  }
  System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes);
  floorDataReader.reset(floorData, 0, numBytes);
  numFollowFloorBlocks = floorDataReader.readVInt();
  nextFloorLabel = floorDataReader.readByte() & 0xff;
  //if (DEBUG) {
  //System.out.println("    setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
  //}
}
Example 5
Source File: FSTCompletionLookup.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Override
public void build(InputIterator iterator) throws IOException {
  if (iterator.hasPayloads()) {
    throw new IllegalArgumentException("this suggester doesn't support payloads");
  }
  if (iterator.hasContexts()) {
    throw new IllegalArgumentException("this suggester doesn't support contexts");
  }

  OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix);
  ExternalRefSorter externalSorter = new ExternalRefSorter(sorter);
  IndexOutput tempInput = tempDir.createTempOutput(tempFileNamePrefix, "input", IOContext.DEFAULT);
  String tempSortedFileName = null;

  OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
  OfflineSorter.ByteSequencesReader reader = null;

  // Push floats up front before sequences to sort them. For now, assume they are non-negative.
  // If negative floats are allowed some trickery needs to be done to find their byte order.
  count = 0;
  try {
    byte [] buffer = new byte [0];
    ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
    BytesRef spare;
    int inputLineCount = 0;
    while ((spare = iterator.next()) != null) {
      if (spare.length + 4 >= buffer.length) {
        buffer = ArrayUtil.grow(buffer, spare.length + 4);
      }

      output.reset(buffer);
      output.writeInt(encodeWeight(iterator.weight()));
      output.writeBytes(spare.bytes, spare.offset, spare.length);
      writer.write(buffer, 0, output.getPosition());
      inputLineCount++;
    }
    CodecUtil.writeFooter(tempInput);
    writer.close();

    // We don't know the distribution of scores and we need to bucket them, so we'll sort
    // and divide into equal buckets.
    tempSortedFileName = sorter.sort(tempInput.getName());
    tempDir.deleteFile(tempInput.getName());

    FSTCompletionBuilder builder = new FSTCompletionBuilder(
        buckets, externalSorter, sharedTailLength);

    reader = new OfflineSorter.ByteSequencesReader(tempDir.openChecksumInput(tempSortedFileName, IOContext.READONCE), tempSortedFileName);
    long line = 0;
    int previousBucket = 0;
    int previousScore = 0;
    ByteArrayDataInput input = new ByteArrayDataInput();
    BytesRef tmp2 = new BytesRef();
    while (true) {
      BytesRef scratch = reader.next();
      if (scratch == null) {
        break;
      }
      input.reset(scratch.bytes, scratch.offset, scratch.length);
      int currentScore = input.readInt();

      int bucket;
      if (line > 0 && currentScore == previousScore) {
        bucket = previousBucket;
      } else {
        bucket = (int) (line * buckets / inputLineCount);
      }
      previousScore = currentScore;
      previousBucket = bucket;

      // Only append the input, discard the weight.
      tmp2.bytes = scratch.bytes;
      tmp2.offset = scratch.offset + input.getPosition();
      tmp2.length = scratch.length - input.getPosition();
      builder.add(tmp2, bucket);

      line++;
      count++;
    }

    // The two FSTCompletions share the same automaton.
    this.higherWeightsCompletion = builder.build();
    this.normalCompletion = new FSTCompletion(
        higherWeightsCompletion.getFST(), false, exactMatchFirst);
    
  } finally {
    IOUtils.closeWhileHandlingException(reader, writer, externalSorter);
    IOUtils.deleteFilesIgnoringExceptions(tempDir, tempInput.getName(), tempSortedFileName);
  }
}