Java Code Examples for org.apache.lucene.store.DataOutput#writeVInt()

The following examples show how to use org.apache.lucene.store.DataOutput#writeVInt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SortingLeafReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void addPositions(final PostingsEnum in, final DataOutput out) throws IOException {
  int freq = in.freq();
  out.writeVInt(freq);
  int previousPosition = 0;
  int previousEndOffset = 0;
  for (int i = 0; i < freq; i++) {
    final int pos = in.nextPosition();
    final BytesRef payload = in.getPayload();
    // The low-order bit of token is set only if there is a payload, the
    // previous bits are the delta-encoded position.
    final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1);
    out.writeVInt(token);
    previousPosition = pos;
    if (storeOffsets) { // don't encode offsets if they are not stored
      final int startOffset = in.startOffset();
      final int endOffset = in.endOffset();
      out.writeVInt(startOffset - previousEndOffset);
      out.writeVInt(endOffset - startOffset);
      previousEndOffset = endOffset;
    }
    if (payload != null) {
      out.writeVInt(payload.length);
      out.writeBytes(payload.bytes, payload.offset, payload.length);
    }
  }
}
 
Example 2
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example 3
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 2) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters");
        }
        out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
      }
    }
  }
}
 
Example 4
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example 5
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte)0);
        out.writeByte((byte)0);
        out.writeByte((byte)0);
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 3) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters");
        }
        out.writeString(data[0]);
        out.writeString(data[1]);
        out.writeString(data[2]);
      }
    }
  }
}
 
Example 6
Source File: XAnalyzingSuggester.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    if (fst == null) {
      return false;
    }

    fst.save(dataOut);
    dataOut.writeVInt(maxAnalyzedPathsForOneInput);
    dataOut.writeByte((byte) (hasPayloads ? 1 : 0));
  } finally {
    IOUtils.close(output);
  }
  return true;
}
 
Example 7
Source File: Lucene50PostingsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IntBlockTermState state = (IntBlockTermState)_state;
  if (absolute) {
    lastState = emptyState;
  }
  out.writeVLong(state.docStartFP - lastState.docStartFP);
  if (writePositions) {
    out.writeVLong(state.posStartFP - lastState.posStartFP);
    if (writePayloads || writeOffsets) {
      out.writeVLong(state.payStartFP - lastState.payStartFP);
    }
  }
  if (state.singletonDocID != -1) {
    out.writeVInt(state.singletonDocID);
  }
  if (writePositions) {
    if (state.lastPosBlockOffset != -1) {
      out.writeVLong(state.lastPosBlockOffset);
    }
  }
  if (state.skipOffset != -1) {
    out.writeVLong(state.skipOffset);
  }
  lastState = state;
}
 
Example 8
Source File: IDVersionPostingsWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IDVersionTermState state = (IDVersionTermState) _state;
  out.writeVInt(state.docID);
  if (absolute) {
    out.writeVLong(state.idVersion);
  } else {
    long delta = state.idVersion - lastEncodedVersion;
    out.writeZLong(delta);
  }
  lastEncodedVersion = state.idVersion;
}
 
Example 9
Source File: BKDWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
  if (numIndexDims != 1) {
    writeActualBounds(out, commonPrefixLengths, count, packedValues);
  }
  BytesRef value = packedValues.apply(0);
  System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
  int cardinality = 1;
  for (int i = 1; i < count; i++) {
    value = packedValues.apply(i);
    for(int dim = 0; dim < numDataDims; dim++) {
      final int start = dim * bytesPerDim + commonPrefixLengths[dim];
      final int end = dim * bytesPerDim + bytesPerDim;
      if (Arrays.mismatch(value.bytes, value.offset + start, value.offset + end, scratch1, start, end) != -1) {
        out.writeVInt(cardinality);
        for (int j = 0; j < numDataDims; j++) {
          out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
        }
        System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
        cardinality = 1;
        break;
      } else if (dim == numDataDims - 1){
        cardinality++;
      }
    }
  }
  out.writeVInt(cardinality);
  for (int i = 0; i < numDataDims; i++) {
    out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
  }
}
 
Example 10
Source File: DocIdsWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOException {
  // docs can be sorted either when all docs in a block have the same value
  // or when a segment is sorted
  boolean sorted = true;
  for (int i = 1; i < count; ++i) {
    if (docIds[start + i - 1] > docIds[start + i]) {
      sorted = false;
      break;
    }
  }
  if (sorted) {
    out.writeByte((byte) 0);
    int previous = 0;
    for (int i = 0; i < count; ++i) {
      int doc = docIds[start + i];
      out.writeVInt(doc - previous);
      previous = doc;
    }
  } else {
    long max = 0;
    for (int i = 0; i < count; ++i) {
      max |= Integer.toUnsignedLong(docIds[start + i]);
    }
    if (max <= 0xffffff) {
      out.writeByte((byte) 24);
      for (int i = 0; i < count; ++i) {
        out.writeShort((short) (docIds[start + i] >>> 8));
        out.writeByte((byte) docIds[start + i]);
      }
    } else {
      out.writeByte((byte) 32);
      for (int i = 0; i < count; ++i) {
        out.writeInt(docIds[start + i]);
      }
    }
  }
}
 
Example 11
Source File: AnalyzingSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }

  fst.save(output, output);
  output.writeVInt(maxAnalyzedPathsForOneInput);
  output.writeByte((byte) (hasPayloads ? 1 : 0));
  return true;
}
 
Example 12
Source File: FreeTextSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
  output.writeVLong(count);
  output.writeByte(separator);
  output.writeVInt(grams);
  output.writeVLong(totTokens);
  fst.save(output, output);
  return true;
}
 
Example 13
Source File: FSTOrdsOutputs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void write(Output prefix, DataOutput out) throws IOException {
  out.writeVInt(prefix.bytes.length);
  out.writeBytes(prefix.bytes.bytes, prefix.bytes.offset, prefix.bytes.length);
  out.writeVLong(prefix.startOrd);
  out.writeVLong(prefix.endOrd);
}
 
Example 14
Source File: SimplePrimaryNode.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void verifyAtLeastMarkerCount(int expectedAtLeastCount, DataOutput out) throws IOException {
  IndexSearcher searcher = mgr.acquire();
  try {
    long version = ((DirectoryReader) searcher.getIndexReader()).getVersion();
    int hitCount = searcher.count(new TermQuery(new Term("marker", "marker")));

    if (hitCount < expectedAtLeastCount) {
      message("marker search: expectedAtLeastCount=" + expectedAtLeastCount + " but hitCount=" + hitCount);
      TopDocs hits = searcher.search(new TermQuery(new Term("marker", "marker")), expectedAtLeastCount);
      List<Integer> seen = new ArrayList<>();
      for(ScoreDoc hit : hits.scoreDocs) {
        Document doc = searcher.doc(hit.doc);
        seen.add(Integer.parseInt(doc.get("docid").substring(1)));
      }
      Collections.sort(seen);
      message("saw markers:");
      for(int marker : seen) {
        message("saw m" + marker);
      }
      throw new IllegalStateException("at flush: marker count " + hitCount + " but expected at least " + expectedAtLeastCount + " version=" + version);
    }

    if (out != null) {
      out.writeVLong(version);
      out.writeVInt(hitCount);
    }
  } finally {
    mgr.release(searcher);
  }
}
 
Example 15
Source File: BKDWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException {
  for(int dim=0;dim<numDataDims;dim++) {
    out.writeVInt(commonPrefixes[dim]);
    //System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
    out.writeBytes(packedValue, dim*bytesPerDim, commonPrefixes[dim]);
  }
}
 
Example 16
Source File: SimpleServer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static void writeFilesMetaData(DataOutput out, Map<String,FileMetaData> files) throws IOException {
  out.writeVInt(files.size());
  for(Map.Entry<String,FileMetaData> ent : files.entrySet()) {
    out.writeString(ent.getKey());

    FileMetaData fmd = ent.getValue();
    out.writeVLong(fmd.length);
    out.writeVLong(fmd.checksum);
    out.writeVInt(fmd.header.length);
    out.writeBytes(fmd.header, 0, fmd.header.length);
    out.writeVInt(fmd.footer.length);
    out.writeBytes(fmd.footer, 0, fmd.footer.length);
  }
}
 
Example 17
Source File: XAnalyzingSuggester.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }

  fst.save(output);
  output.writeVInt(maxAnalyzedPathsForOneInput);
  output.writeByte((byte) (hasPayloads ? 1 : 0));
  return true;
}
 
Example 18
Source File: BlockTreeTermsWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private static void writeBytesRef(DataOutput out, BytesRef bytes) throws IOException {
  out.writeVInt(bytes.length);
  out.writeBytes(bytes.bytes, bytes.offset, bytes.length);
}
 
Example 19
Source File: DocValuesUpdate.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
void writeTo(DataOutput out) throws IOException {
  assert hasValue;
  out.writeVInt(value.length);
  out.writeBytes(value.bytes, value.offset, value.length);
}
 
Example 20
Source File: BlockLine.java    From lucene-solr with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a line and its offset to the corresponding term state details in
 * the details region.
 *
 * @param blockOutput               The output pointing to the block terms region.
 * @param termStateRelativeOffset   The offset to the corresponding term
 *                                  state details in the details region.
 * @param isIncrementalEncodingSeed Whether the term is a seed of
 *                                  the incremental encoding. {@code true} for the first
 *                                  and middle term, {@code false} for other terms.
 */
public void writeLine(DataOutput blockOutput, BlockLine line, BlockLine previousLine,
                             int termStateRelativeOffset, boolean isIncrementalEncodingSeed) throws IOException {
  blockOutput.writeVInt(termStateRelativeOffset);
  writeIncrementallyEncodedTerm(line.getTermBytes(), previousLine == null ? null : previousLine.getTermBytes(),
      isIncrementalEncodingSeed, blockOutput);
}