Java Code Examples for org.apache.lucene.store.DataOutput

The following examples show how to use org.apache.lucene.store.DataOutput. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: Lucene50PostingsWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IntBlockTermState state = (IntBlockTermState)_state;
  if (absolute) {
    lastState = emptyState;
  }
  out.writeVLong(state.docStartFP - lastState.docStartFP);
  if (writePositions) {
    out.writeVLong(state.posStartFP - lastState.posStartFP);
    if (writePayloads || writeOffsets) {
      out.writeVLong(state.payStartFP - lastState.payStartFP);
    }
  }
  if (state.singletonDocID != -1) {
    out.writeVInt(state.singletonDocID);
  }
  if (writePositions) {
    if (state.lastPosBlockOffset != -1) {
      out.writeVLong(state.lastPosBlockOffset);
    }
  }
  if (state.skipOffset != -1) {
    out.writeVLong(state.skipOffset);
  }
  lastState = state;
}
 
Example 2
Source Project: lucene-solr   Source File: SimplePrimaryNode.java    License: Apache License 2.0 6 votes vote down vote up
private void handleAddDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid") || name.equals("marker")) {
      doc.add(new StringField(name, value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }
  writer.addDocument(doc);
}
 
Example 3
Source Project: lucene-solr   Source File: SimplePrimaryNode.java    License: Apache License 2.0 6 votes vote down vote up
private void handleUpdateDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  String docid = null;
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid")) {
      docid = value;
      doc.add(new StringField("docid", value, Field.Store.YES));
    } else if (name.equals("marker")) {
      doc.add(new StringField("marker", value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }

  writer.updateDocument(new Term("docid", docid), doc);
}
 
Example 4
Source Project: lucene-solr   Source File: LZ4.java    License: Apache License 2.0 6 votes vote down vote up
private static void encodeSequence(byte[] bytes, int anchor, int matchRef, int matchOff, int matchLen, DataOutput out) throws IOException {
  final int literalLen = matchOff - anchor;
  assert matchLen >= 4;
  // encode token
  final int token = (Math.min(literalLen, 0x0F) << 4) | Math.min(matchLen - 4, 0x0F);
  encodeLiterals(bytes, token, anchor, literalLen, out);

  // encode match dec
  final int matchDec = matchOff - matchRef;
  assert matchDec > 0 && matchDec < 1 << 16;
  out.writeByte((byte) matchDec);
  out.writeByte((byte) (matchDec >>> 8));

  // encode match len
  if (matchLen >= MIN_MATCH + 0x0F) {
    encodeLen(matchLen - 0x0F - MIN_MATCH, out);
  }
}
 
Example 5
Source Project: lucene-solr   Source File: JaspellLookup.java    License: Apache License 2.0 6 votes vote down vote up
private void writeRecursively(DataOutput out, TSTNode node) throws IOException {
  if (node == null) {
    return;
  }
  out.writeString(new String(new char[] {node.splitchar}, 0, 1));
  byte mask = 0;
  if (node.relatives[TSTNode.LOKID] != null) mask |= LO_KID;
  if (node.relatives[TSTNode.EQKID] != null) mask |= EQ_KID;
  if (node.relatives[TSTNode.HIKID] != null) mask |= HI_KID;
  if (node.data != null) mask |= HAS_VALUE;
  out.writeByte(mask);
  if (node.data != null) {
    out.writeLong(((Number)node.data).longValue());
  }
  writeRecursively(out, node.relatives[TSTNode.LOKID]);
  writeRecursively(out, node.relatives[TSTNode.EQKID]);
  writeRecursively(out, node.relatives[TSTNode.HIKID]);
}
 
Example 6
Source Project: lucene-solr   Source File: ByteSliceReader.java    License: Apache License 2.0 6 votes vote down vote up
public long writeTo(DataOutput out) throws IOException {
  long size = 0;
  while(true) {
    if (limit + bufferOffset == endIndex) {
      assert endIndex - bufferOffset >= upto;
      out.writeBytes(buffer, upto, limit-upto);
      size += limit-upto;
      break;
    } else {
      out.writeBytes(buffer, upto, limit-upto);
      size += limit-upto;
      nextSlice();
    }
  }

  return size;
}
 
Example 7
Source Project: lucene-solr   Source File: BinaryDictionaryWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte)0);
        out.writeByte((byte)0);
        out.writeByte((byte)0);
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 3) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters");
        }
        out.writeString(data[0]);
        out.writeString(data[1]);
        out.writeString(data[2]);
      }
    }
  }
}
 
Example 8
Source Project: lucene-solr   Source File: CharacterDefinitionWriter.java    License: Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example 9
Source Project: lucene-solr   Source File: BinaryDictionaryWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 2) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters");
        }
        out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
      }
    }
  }
}
 
Example 10
Source Project: lucene-solr   Source File: CharacterDefinitionWriter.java    License: Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example 11
Source Project: lucene-solr   Source File: ConnectionCostsWriter.java    License: Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example 12
protected BlockEncoder getBlockEncoder() {
  return (blockBytes, length) -> {
    byte[] encodedBytes = Rot13CypherTestUtil.encode(blockBytes, Math.toIntExact(length));
    return new BlockEncoder.WritableBytes() {
      @Override
      public long size() {
        return encodedBytes.length;
      }

      @Override
      public void writeTo(DataOutput dataOutput) throws IOException {
        encoderCalled = true;
        dataOutput.writeBytes(encodedBytes, 0, encodedBytes.length);
      }
    };
  };
}
 
Example 13
Source Project: lucene-solr   Source File: ForUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a new {@link ForUtil} instance and save state into <code>out</code>.
 */
ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException {
  out.writeVInt(PackedInts.VERSION_CURRENT);
  encodedSizes = new int[33];
  encoders = new PackedInts.Encoder[33];
  decoders = new PackedInts.Decoder[33];
  iterations = new int[33];

  for (int bpv = 1; bpv <= 32; ++bpv) {
    final FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(
        BLOCK_SIZE, bpv, acceptableOverheadRatio);
    assert formatAndBits.format.isSupported(formatAndBits.bitsPerValue);
    assert formatAndBits.bitsPerValue <= 32;
    encodedSizes[bpv] = encodedSize(formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
    encoders[bpv] = PackedInts.getEncoder(
        formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
    decoders[bpv] = PackedInts.getDecoder(
        formatAndBits.format, PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue);
    iterations[bpv] = computeIterations(decoders[bpv]);

    out.writeVInt(formatAndBits.format.getId() << 5 | (formatAndBits.bitsPerValue - 1));
  }
}
 
Example 14
Source Project: lucene-solr   Source File: Lucene84SkipWriter.java    License: Apache License 2.0 6 votes vote down vote up
static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out) throws IOException {
  Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();
  Impact previous = new Impact(0, 0);
  for (Impact impact : impacts) {
    assert impact.freq > previous.freq;
    assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
    int freqDelta = impact.freq - previous.freq - 1;
    long normDelta = impact.norm - previous.norm - 1;
    if (normDelta == 0) {
      // most of time, norm only increases by 1, so we can fold everything in a single byte
      out.writeVInt(freqDelta << 1);
    } else {
      out.writeVInt((freqDelta << 1) | 1);
      out.writeZLong(normDelta);
    }
    previous = impact;
  }
}
 
Example 15
Source Project: Elasticsearch   Source File: XAnalyzingSuggester.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }

  fst.save(output);
  output.writeVInt(maxAnalyzedPathsForOneInput);
  output.writeByte((byte) (hasPayloads ? 1 : 0));
  return true;
}
 
Example 16
Source Project: Elasticsearch   Source File: BloomFilter.java    License: Apache License 2.0 5 votes vote down vote up
public static void serilaize(BloomFilter filter, DataOutput out) throws IOException {
    out.writeInt(0); // version
    BitArray bits = filter.bits;
    out.writeInt(bits.data.length);
    for (long l : bits.data) {
        out.writeLong(l);
    }
    out.writeInt(filter.numHashFunctions);
    out.writeInt(filter.hashing.type()); // hashType
}
 
Example 17
Source Project: lucene-solr   Source File: SimpleServer.java    License: Apache License 2.0 5 votes vote down vote up
static void writeFilesMetaData(DataOutput out, Map<String,FileMetaData> files) throws IOException {
  out.writeVInt(files.size());
  for(Map.Entry<String,FileMetaData> ent : files.entrySet()) {
    out.writeString(ent.getKey());

    FileMetaData fmd = ent.getValue();
    out.writeVLong(fmd.length);
    out.writeVLong(fmd.checksum);
    out.writeVInt(fmd.header.length);
    out.writeBytes(fmd.header, 0, fmd.header.length);
    out.writeVInt(fmd.footer.length);
    out.writeBytes(fmd.footer, 0, fmd.footer.length);
  }
}
 
Example 18
Source Project: lucene-solr   Source File: SimplePrimaryNode.java    License: Apache License 2.0 5 votes vote down vote up
/** Pushes CopyState on the wire */
private static void writeCopyState(CopyState state, DataOutput out) throws IOException {
  // TODO (opto): we could encode to byte[] once when we created the copyState, and then just send same byts to all replicas...
  out.writeVInt(state.infosBytes.length);
  out.writeBytes(state.infosBytes, 0, state.infosBytes.length);
  out.writeVLong(state.gen);
  out.writeVLong(state.version);
  SimpleServer.writeFilesMetaData(out, state.files);

  out.writeVInt(state.completedMergeFiles.size());
  for(String fileName : state.completedMergeFiles) {
    out.writeString(fileName);
  }
  out.writeVLong(state.primaryGen);
}
 
Example 19
Source Project: lucene-solr   Source File: FSTTermOutputs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(TermData data, DataOutput out) throws IOException {
  assert hasPos || data.totalTermFreq == -1;
  int bit0 = ((data.bytes == null || data.bytes.length == 0) ? 0 : 1);
  int bit1 = ((data.docFreq == 0)  ? 0 : 1) << 1;
  int bits = bit0 | bit1;
  if (bit0 > 0) {  // determine extra length
    if (data.bytes.length < 32) {
      bits |= (data.bytes.length << 2);
      out.writeByte((byte)bits);
    } else {
      out.writeByte((byte)bits);
      out.writeVInt(data.bytes.length);
    }
  } else {
    out.writeByte((byte)bits);
  }
  if (bit0 > 0) {  // bytes exists
    out.writeBytes(data.bytes, 0, data.bytes.length);
  }
  if (bit1 > 0) {  // stats exist
    if (hasPos) {
      if (data.docFreq == data.totalTermFreq) {
        out.writeVInt((data.docFreq << 1) | 1);
      } else {
        out.writeVInt((data.docFreq << 1));
        out.writeVLong(data.totalTermFreq - data.docFreq);
      }
    } else {
      out.writeVInt(data.docFreq);
    }
  }
}
 
Example 20
Source Project: lucene-solr   Source File: FSTOrdsOutputs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(Output prefix, DataOutput out) throws IOException {
  out.writeVInt(prefix.bytes.length);
  out.writeBytes(prefix.bytes.bytes, prefix.bytes.offset, prefix.bytes.length);
  out.writeVLong(prefix.startOrd);
  out.writeVLong(prefix.endOrd);
}
 
Example 21
Source Project: lucene-solr   Source File: OnHeapFSTStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeTo(DataOutput out) throws IOException {
    if (bytes != null) {
        long numBytes = bytes.getPosition();
        out.writeVLong(numBytes);
        bytes.writeTo(out);
    } else {
        assert bytesArray != null;
        out.writeVLong(bytesArray.length);
        out.writeBytes(bytesArray, 0, bytesArray.length);
    }
}
 
Example 22
Source Project: lucene-solr   Source File: FSTDictionary.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutput output, BlockEncoder blockEncoder) throws IOException {
  if (blockEncoder == null) {
    fst.save(output, output);
  } else {
    ByteBuffersDataOutput bytesDataOutput = ByteBuffersDataOutput.newResettableInstance();
    fst.save(bytesDataOutput, bytesDataOutput);
    BlockEncoder.WritableBytes encodedBytes = blockEncoder.encode(bytesDataOutput.toDataInput(), bytesDataOutput.size());
    output.writeVLong(encodedBytes.size());
    encodedBytes.writeTo(output);
  }
}
 
Example 23
Source Project: lucene-solr   Source File: UniformSplitTermsWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return 1 if the field was written; 0 otherwise.
 */
protected int writeFieldTerms(BlockWriter blockWriter, DataOutput fieldsOutput, TermsEnum termsEnum,
                            FieldInfo fieldInfo, NormsProducer normsProducer) throws IOException {

  FieldMetadata fieldMetadata = new FieldMetadata(fieldInfo, maxDoc);
  fieldMetadata.setDictionaryStartFP(dictionaryOutput.getFilePointer());

  postingsWriter.setField(fieldInfo);
  blockWriter.setField(fieldMetadata);
  IndexDictionary.Builder dictionaryBuilder = new FSTDictionary.Builder();
  BytesRef lastTerm = null;
  while (termsEnum.next() != null) {
    BlockTermState blockTermState = writePostingLine(termsEnum, fieldMetadata, normsProducer);
    if (blockTermState != null) {
      lastTerm = BytesRef.deepCopyOf(termsEnum.term());
      blockWriter.addLine(lastTerm, blockTermState, dictionaryBuilder);
    }
  }

  // Flush remaining terms.
  blockWriter.finishLastBlock(dictionaryBuilder);

  if (fieldMetadata.getNumTerms() > 0) {
    fieldMetadata.setLastTerm(lastTerm);
    fieldMetadataWriter.write(fieldsOutput, fieldMetadata);
    writeDictionary(dictionaryBuilder);
    return 1;
  }
  return 0;
}
 
Example 24
Source Project: lucene-solr   Source File: FreeTextSuggester.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
  output.writeVLong(count);
  output.writeByte(separator);
  output.writeVInt(grams);
  output.writeVLong(totTokens);
  fst.save(output, output);
  return true;
}
 
Example 25
Source Project: lucene-solr   Source File: DirectWriter.java    License: Apache License 2.0 5 votes vote down vote up
DirectWriter(DataOutput output, long numValues, int bitsPerValue) {
  this.output = output;
  this.numValues = numValues;
  this.bitsPerValue = bitsPerValue;
  encoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
  iterations = encoder.computeIterations((int) Math.min(numValues, Integer.MAX_VALUE), PackedInts.DEFAULT_BUFFER_SIZE);
  nextBlocks = new byte[iterations * encoder.byteBlockCount()];
  nextValues = new long[iterations * encoder.byteValueCount()];
}
 
Example 26
Source Project: lucene-solr   Source File: Lookup.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calls {@link #store(DataOutput)} after converting
 * {@link OutputStream} to {@link DataOutput}
 */
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    return store(dataOut);
  } finally {
    IOUtils.close(output);
  }
}
 
Example 27
Source Project: lucene-solr   Source File: PackedWriter.java    License: Apache License 2.0 5 votes vote down vote up
PackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem) {
  super(out, valueCount, bitsPerValue);
  this.format = format;
  encoder = BulkOperation.of(format, bitsPerValue);
  iterations = encoder.computeIterations(valueCount, mem);
  nextBlocks = new byte[iterations * encoder.byteBlockCount()];
  nextValues = new long[iterations * encoder.byteValueCount()];
  off = 0;
  written = 0;
  finished = false;
}
 
Example 28
Source Project: lucene-solr   Source File: NRTSuggesterBuilder.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Builds and stores a FST that can be loaded with
 * {@link NRTSuggester#load(IndexInput, CompletionPostingsFormat.FSTLoadMode)})}
 */
public boolean store(DataOutput output) throws IOException {
  final FST<PairOutputs.Pair<Long, BytesRef>> fst = fstCompiler.compile();
  if (fst == null) {
    return false;
  }
  fst.save(output, output);

  /* write some more  meta-info */
  assert maxAnalyzedPathsPerOutput > 0;
  output.writeVInt(maxAnalyzedPathsPerOutput);
  output.writeVInt(END_BYTE);
  output.writeVInt(PAYLOAD_SEP);
  return true;
}
 
Example 29
Source Project: lucene-solr   Source File: IDVersionPostingsWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IDVersionTermState state = (IDVersionTermState) _state;
  out.writeVInt(state.docID);
  if (absolute) {
    out.writeVLong(state.idVersion);
  } else {
    long delta = state.idVersion - lastEncodedVersion;
    out.writeZLong(delta);
  }
  lastEncodedVersion = state.idVersion;
}
 
Example 30
Source Project: lucene-solr   Source File: IntSequenceOutputs.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(IntsRef prefix, DataOutput out) throws IOException {
  assert prefix != null;
  out.writeVInt(prefix.length);
  for(int idx=0;idx<prefix.length;idx++) {
    out.writeVInt(prefix.ints[prefix.offset+idx]);
  }
}