Java Code Examples for org.apache.lucene.store.DataOutput#writeBytes()

public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 

public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 

protected BlockEncoder getBlockEncoder() {
  return (blockBytes, length) -> {
    byte[] encodedBytes = Rot13CypherTestUtil.encode(blockBytes, Math.toIntExact(length));
    return new BlockEncoder.WritableBytes() {
      @Override
      public long size() {
        return encodedBytes.length;
      }

      @Override
      public void writeTo(DataOutput dataOutput) throws IOException {
        encoderCalled = true;
        dataOutput.writeBytes(encodedBytes, 0, encodedBytes.length);
      }
    };
  };
}
 

public static BlockEncoder getBlockEncoder() {
  return (blockBytes, length) -> {
    byte[] encodedBytes = Rot13CypherTestUtil.encode(blockBytes, Math.toIntExact(length));
    return new BlockEncoder.WritableBytes() {
      @Override
      public long size() {
        return encodedBytes.length;
      }

      @Override
      public void writeTo(DataOutput dataOutput) throws IOException {
        dataOutput.writeBytes(encodedBytes, 0, encodedBytes.length);
      }
    };
  };
}
 

private void addPositions(final PostingsEnum in, final DataOutput out) throws IOException {
  int freq = in.freq();
  out.writeVInt(freq);
  int previousPosition = 0;
  int previousEndOffset = 0;
  for (int i = 0; i < freq; i++) {
    final int pos = in.nextPosition();
    final BytesRef payload = in.getPayload();
    // The low-order bit of token is set only if there is a payload, the
    // previous bits are the delta-encoded position.
    final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1);
    out.writeVInt(token);
    previousPosition = pos;
    if (storeOffsets) { // don't encode offsets if they are not stored
      final int startOffset = in.startOffset();
      final int endOffset = in.endOffset();
      out.writeVInt(startOffset - previousEndOffset);
      out.writeVInt(endOffset - startOffset);
      previousEndOffset = endOffset;
    }
    if (payload != null) {
      out.writeVInt(payload.length);
      out.writeBytes(payload.bytes, payload.offset, payload.length);
    }
  }
}
 

/** Pushes CopyState on the wire */
private static void writeCopyState(CopyState state, DataOutput out) throws IOException {
  // TODO (opto): we could encode to byte[] once when we created the copyState, and then just send same byts to all replicas...
  out.writeVInt(state.infosBytes.length);
  out.writeBytes(state.infosBytes, 0, state.infosBytes.length);
  out.writeVLong(state.gen);
  out.writeVLong(state.version);
  SimpleServer.writeFilesMetaData(out, state.files);

  out.writeVInt(state.completedMergeFiles.size());
  for(String fileName : state.completedMergeFiles) {
    out.writeString(fileName);
  }
  out.writeVLong(state.primaryGen);
}
 

static void writeFilesMetaData(DataOutput out, Map<String,FileMetaData> files) throws IOException {
  out.writeVInt(files.size());
  for(Map.Entry<String,FileMetaData> ent : files.entrySet()) {
    out.writeString(ent.getKey());

    FileMetaData fmd = ent.getValue();
    out.writeVLong(fmd.length);
    out.writeVLong(fmd.checksum);
    out.writeVInt(fmd.header.length);
    out.writeBytes(fmd.header, 0, fmd.header.length);
    out.writeVInt(fmd.footer.length);
    out.writeBytes(fmd.footer, 0, fmd.footer.length);
  }
}
 

@Override
public void write(Output prefix, DataOutput out) throws IOException {
  out.writeVInt(prefix.bytes.length);
  out.writeBytes(prefix.bytes.bytes, prefix.bytes.offset, prefix.bytes.length);
  out.writeVLong(prefix.startOrd);
  out.writeVLong(prefix.endOrd);
}
 

private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException {
  for(int dim=0;dim<numDataDims;dim++) {
    out.writeVInt(commonPrefixes[dim]);
    //System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
    out.writeBytes(packedValue, dim*bytesPerDim, commonPrefixes[dim]);
  }
}
 

private void writeActualBounds(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
  for (int dim = 0; dim < numIndexDims; ++dim) {
    int commonPrefixLength = commonPrefixLengths[dim];
    int suffixLength = bytesPerDim - commonPrefixLength;
    if (suffixLength > 0) {
      BytesRef[] minMax = computeMinMax(count, packedValues, dim * bytesPerDim + commonPrefixLength, suffixLength);
      BytesRef min = minMax[0];
      BytesRef max = minMax[1];
      out.writeBytes(min.bytes, min.offset, min.length);
      out.writeBytes(max.bytes, max.offset, max.length);
    }
  }
}
 

/**
 * Expert: verifies the incoming {@link IndexInput} has an index header
 * and that its segment ID matches the expected one, and then copies
 * that index header into the provided {@link DataOutput}.  This is
 * useful when building compound files.
 *
 * @param in Input stream, positioned at the point where the
 *        index header was previously written. Typically this is located
 *        at the beginning of the file.
 * @param out Output stream, where the header will be copied to.
 * @param expectedID Expected segment ID
 * @throws CorruptIndexException If the first four bytes are not
 *         {@link #CODEC_MAGIC}, or if the <code>expectedID</code>
 *         does not match.
 * @throws IOException If there is an I/O error reading from the underlying medium.
 *
 * @lucene.internal 
 */
public static void verifyAndCopyIndexHeader(IndexInput in, DataOutput out, byte[] expectedID) throws IOException {
  // make sure it's large enough to have a header and footer
  if (in.length() < footerLength() + headerLength("")) {
    throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: file is too small (" + in.length() + " bytes)", in);
  }

  int actualHeader = in.readInt();
  if (actualHeader != CODEC_MAGIC) {
    throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CodecUtil.CODEC_MAGIC, in);
  }

  // we can't verify these, so we pass-through:
  String codec = in.readString();
  int version = in.readInt();

  // verify id:
  checkIndexHeaderID(in, expectedID);

  // we can't verify extension either, so we pass-through:
  int suffixLength = in.readByte() & 0xFF;
  byte[] suffixBytes = new byte[suffixLength];
  in.readBytes(suffixBytes, 0, suffixLength);

  // now write the header we just verified
  out.writeInt(CodecUtil.CODEC_MAGIC);
  out.writeString(codec);
  out.writeInt(version);
  out.writeBytes(expectedID, 0, expectedID.length);
  out.writeByte((byte) suffixLength);
  out.writeBytes(suffixBytes, 0, suffixLength);
}
 

private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
  if (numIndexDims != 1) {
    writeActualBounds(out, commonPrefixLengths, count, packedValues);
  }
  BytesRef value = packedValues.apply(0);
  System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
  int cardinality = 1;
  for (int i = 1; i < count; i++) {
    value = packedValues.apply(i);
    for(int dim = 0; dim < numDataDims; dim++) {
      final int start = dim * bytesPerDim + commonPrefixLengths[dim];
      final int end = dim * bytesPerDim + bytesPerDim;
      if (Arrays.mismatch(value.bytes, value.offset + start, value.offset + end, scratch1, start, end) != -1) {
        out.writeVInt(cardinality);
        for (int j = 0; j < numDataDims; j++) {
          out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
        }
        System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
        cardinality = 1;
        break;
      } else if (dim == numDataDims - 1){
        cardinality++;
      }
    }
  }
  out.writeVInt(cardinality);
  for (int i = 0; i < numDataDims; i++) {
    out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
  }
}
 

private static void encodeLiterals(byte[] bytes, int token, int anchor, int literalLen, DataOutput out) throws IOException {
  out.writeByte((byte) token);

  // encode literal length
  if (literalLen >= 0x0F) {
    encodeLen(literalLen - 0x0F, out);
  }

  // encode literals
  out.writeBytes(bytes, anchor, literalLen);
}
 

@Override
public void writeBytes(DataOutput out, int offset, int length) throws IOException {
  out.writeBytes(_bytes, offset, length);
}
 

/** Writes all of our bytes to the target {@link DataOutput}. */
public void writeTo(DataOutput out) throws IOException {
  for(byte[] block : blocks) {
    out.writeBytes(block, 0, block.length);
  }
}
 

@Override
void writeTo(DataOutput out) throws IOException {
  assert hasValue;
  out.writeVInt(value.length);
  out.writeBytes(value.bytes, value.offset, value.length);
}
 

private static void writeBytesRef(DataOutput out, BytesRef bytes) throws IOException {
  out.writeVInt(bytes.length);
  out.writeBytes(bytes.bytes, bytes.offset, bytes.length);
}
 

/** Called when another node (replica) wants to copy files from us */
private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException {
  Thread.currentThread().setName("send");

  int replicaID = destIn.readVInt();
  message("top: start fetch for R" + replicaID + " socket=" + socket);
  byte b = destIn.readByte();
  CopyState copyState;
  if (b == 0) {
    // Caller already has CopyState
    copyState = null;
  } else if (b == 1) {
    // Caller does not have CopyState; we pull the latest one:
    copyState = getCopyState();
    Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version);
  } else {
    // Protocol error:
    throw new IllegalArgumentException("invalid CopyState byte=" + b);
  }

  try {
    if (copyState != null) {
      // Serialize CopyState on the wire to the client:
      writeCopyState(copyState, destOut);
      bos.flush();
    }

    byte[] buffer = new byte[16384];
    int fileCount = 0;
    long totBytesSent = 0;
    while (true) {
      byte done = destIn.readByte();
      if (done == 1) {
        break;
      } else if (done != 0) {
        throw new IllegalArgumentException("expected 0 or 1 byte but got " + done);
      }

      // Name of the file the replica wants us to send:
      String fileName = destIn.readString();

      // Starting offset in the file we should start sending bytes from:
      long fpStart = destIn.readVLong();

      try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
        long len = in.length();
        //message("fetch " + fileName + ": send len=" + len);
        destOut.writeVLong(len);
        in.seek(fpStart);
        long upto = fpStart;
        while (upto < len) {
          int chunk = (int) Math.min(buffer.length, (len-upto));
          in.readBytes(buffer, 0, chunk);
          if (doFlipBitsDuringCopy) {
            if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) {
              bitFlipped.add(fileName);
              message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto);
              int x = random.nextInt(chunk);
              int bit = random.nextInt(8);
              buffer[x] ^= 1 << bit;
            }
          }
          destOut.writeBytes(buffer, 0, chunk);
          upto += chunk;
          totBytesSent += chunk;
        }
      }

      fileCount++;
    }

    message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes");
  } catch (Throwable t) {
    message("top: exception during fetch: " + t.getMessage() + "; now close socket");
    socket.close();
    return false;
  } finally {
    if (copyState != null) {
      message("top: fetch: now release CopyState");
      releaseCopyState(copyState);
    }
  }

  return true;
}
 

/**
 * Writes a codec header for an index file, which records both a string to
 * identify the format of the file, a version number, and data to identify
 * the file instance (ID and auxiliary suffix such as generation).
 * <p>
 * This header can be parsed and validated with 
 * {@link #checkIndexHeader(DataInput, String, int, int, byte[], String) checkIndexHeader()}.
 * <p>
 * IndexHeader --&gt; CodecHeader,ObjectID,ObjectSuffix
 * <ul>
 *    <li>CodecHeader   --&gt; {@link #writeHeader}
 *    <li>ObjectID     --&gt; {@link DataOutput#writeByte byte}<sup>16</sup>
 *    <li>ObjectSuffix --&gt; SuffixLength,SuffixBytes
 *    <li>SuffixLength  --&gt; {@link DataOutput#writeByte byte}
 *    <li>SuffixBytes   --&gt; {@link DataOutput#writeByte byte}<sup>SuffixLength</sup>
 * </ul>
 * <p>
 * Note that the length of an index header depends only upon the
 * name of the codec and suffix, so this length can be computed at any time
 * with {@link #indexHeaderLength(String,String)}.
 * 
 * @param out Output stream
 * @param codec String to identify the format of this file. It should be simple ASCII, 
 *              less than 128 characters in length.
 * @param id Unique identifier for this particular file instance.
 * @param suffix auxiliary suffix information for the file. It should be simple ASCII,
 *              less than 256 characters in length.
 * @param version Version number
 * @throws IOException If there is an I/O error writing to the underlying medium.
 * @throws IllegalArgumentException If the codec name is not simple ASCII, or 
 *         is more than 127 characters in length, or if id is invalid,
 *         or if the suffix is not simple ASCII, or more than 255 characters
 *         in length.
 */
public static void writeIndexHeader(DataOutput out, String codec, int version, byte[] id, String suffix) throws IOException {
  if (id.length != StringHelper.ID_LENGTH) {
    throw new IllegalArgumentException("Invalid id: " + StringHelper.idToString(id));
  }
  writeHeader(out, codec, version);
  out.writeBytes(id, 0, id.length);
  BytesRef suffixBytes = new BytesRef(suffix);
  if (suffixBytes.length != suffix.length() || suffixBytes.length >= 256) {
    throw new IllegalArgumentException("suffix must be simple ASCII, less than 256 characters in length [got " + suffix + "]");
  }
  out.writeByte((byte) suffixBytes.length);
  out.writeBytes(suffixBytes.bytes, suffixBytes.offset, suffixBytes.length);
}