org.msgpack.core.MessagePack Java Examples

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithoutAckResponse(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(1, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertNull(exception);
}
 

@Test
public void testString() throws IOException
{

    String string = "ч";

    byte[] data = string.getBytes(MessagePack.UTF8);
    // System.out.println(Arrays.toString(data));

    LinkedBuffer lb = LinkedBuffer.allocate();
    WriteSession session = new WriteSession(lb);
    StringSerializer.writeUTF8((CharSequence) string, session, session.tail);

    byte[] xdata = session.toByteArray();
    // System.out.println(Arrays.toString(xdata));

    Assert.assertTrue(Arrays.equals(data, xdata));
}
 

/**
 * Serializes the {@code messages} into the stream using the given schema.
 */
public static <T> void writeListTo(MessageBufferOutput out, List<T> messages,
        Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeListTo(packer, messages, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

/**
 * Serializes the {@code messages} into the stream using the given schema.
 */
public static <T> void writeListTo(OutputStream out, List<T> messages,
        Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeListTo(packer, messages, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

/**
 * Serializes the {@code message} into an {@link MessageBufferOutput} using the given {@code schema}.
 */
public static <T> void writeTo(MessageBufferOutput out, T message, Schema<T> schema, boolean numeric)
        throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeTo(packer, message, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

/**
 * Merges the {@code message} with the byte array using the given {@code schema}.
 */
public static <T> void mergeFrom(byte[] data, int offset, int length, T message, Schema<T> schema, boolean numeric)
        throws IOException
{

    ArrayBufferInput bios = new ArrayBufferInput(data, offset, length);

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bios);

    try
    {
        mergeFrom(unpacker, message, schema, numeric);
    }
    finally
    {
        unpacker.close();
    }
}
 

/**
 * Traverse over the whole message pack stream.
 * This method can be used for converting query results in chunk.
 *
 * @param is
 *          The MessagePack format input stream
 * @return an Iterable over the QueryResult objects
 *
 */
public Iterable<QueryResult> traverse(final InputStream is) {
  MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(is);

  return () -> {
    return new Iterator<QueryResult>() {
      @Override
      public boolean hasNext() {
        try {
          return unpacker.hasNext();
        } catch (IOException e) {
          throw new InfluxDBException(e);
        }
      }

      @Override
      public QueryResult next() {
        return parse(unpacker);
      }
    };
  };

}
 

@Override
public final byte[] build(DataDstImpl compiler) throws ConvException {
    MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();

    DataDstJava.DataDstObject data_obj = build_data(compiler);

    try {
        packer.packMapHeader(3);
        packer.packString("header");
        writeData(packer, data_obj.header);
        packer.packString("data_block");
        writeData(packer, data_obj.body);
        packer.packString("data_message_type");
        writeData(packer, data_obj.data_message_type);
    } catch (IOException e) {
        this.logErrorMessage("MessagePacker write failed.");
        e.printStackTrace();
    }

    return packer.toByteArray();
}
 

public static MaestroEvent deserializeEvent(byte[] bytes) throws IOException, MalformedNoteException {
    try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)){
        final short tmpType = unpacker.unpackShort();
        final MaestroNoteType type = MaestroNoteType.from(tmpType);

        switch (Objects.requireNonNull(type)) {
            case MAESTRO_TYPE_REQUEST:
                return deserializeRequest(unpacker);
            case MAESTRO_TYPE_NOTIFICATION:
                return deserializeNotification(unpacker);
            case MAESTRO_TYPE_DATA:
                return deserializeData(unpacker);
            default:
                throw new MalformedNoteException("Invalid event type: " + tmpType);
        }
    }
}
 

@Test
void ingestWithoutAck()
        throws IOException
{
    Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender);
    ingester.ingest(TAG, ByteBuffer.wrap(DATA));

    verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture());
    List<ByteBuffer> byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0);
    byte[] ingested = getIngestedData(byteBuffers);

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested);
    ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue();
    assertEquals(3, arrayValue.size());
    assertEquals(TAG, arrayValue.get(0).asStringValue().asString());
    assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray());
    Map<Value, Value> options = arrayValue.get(2).asMapValue().map();
    assertEquals(1, options.size());
    assertEquals(DATA.length,
            options.get(ValueFactory.newString("size")).asIntegerValue().asInt());
}
 

@Override
public void setTimestamp(long timestamp)
{
    int mapSize = mapValueBytes.map().size();
    try (ByteArrayOutputStream output = new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) {
        try (MessagePacker packer = MessagePack.newDefaultPacker(output)) {
            if (mapValueBytes.map().containsKey(KEY_TIME)) {
                packer.packMapHeader(mapSize);
            }
            else {
                packer.packMapHeader(mapSize + 1);
                KEY_TIME.writeTo(packer);
                packer.packLong(timestamp);
            }

            for (Map.Entry<Value, Value> entry : mapValueBytes.map().entrySet()) {
                packer.packValue(entry.getKey());
                packer.packValue(entry.getValue());
            }
        }
        mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray()));
    }
    catch (IOException e) {
        throw new IllegalStateException("Failed to upsert `time` field", e);
    }
}
 

@Override
public <T> T deserialize( ModuleDescriptor module, ValueType valueType, InputStream state )
{
    try( MessageUnpacker unpacker = MessagePack.newDefaultUnpacker( state ) )
    {
        if( !unpacker.hasNext() )
        {
            return null;
        }
        ImmutableValue value = unpacker.unpackValue();
        return doDeserialize( module, valueType, value );
    }
    catch( IOException ex )
    {
        throw new SerializationException( "Unable to deserialize " + valueType, ex );
    }
}
 

public static MaestroNote deserialize(byte[] bytes) throws IOException, MalformedNoteException {
    try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)){
        final short tmpType = unpacker.unpackShort();
        final MaestroNoteType type = MaestroNoteType.from(tmpType);

        switch (Objects.requireNonNull(type)) {
            case MAESTRO_TYPE_REQUEST:
                return deserializeRequest(unpacker);
            case MAESTRO_TYPE_RESPONSE:
                return deserializeResponse(unpacker);
            case MAESTRO_TYPE_NOTIFICATION:
                return deserializeNotification(unpacker);
            case MAESTRO_TYPE_DATA:
                return deserializeData(unpacker);
            default:
                throw new MalformedNoteException("Invalid note type: " + tmpType);
        }
    }
}
 

protected MessageBufferPacker pack() throws IOException {
    MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();

    packer.packShort(noteType.getValue());
    packer.packLong(maestroCommand.getValue());

    correlate();

    packer.packString(correlation.getCorrelationId());
    packer.packString(correlation.getMessageId());

    if (logger.isTraceEnabled()) {
        logger.trace("Packed this {} note with correlation id {} and message id {}", this.maestroCommand,
                correlation.getCorrelationId(), correlation.getMessageId());
    }

    return packer;
}
 

public byte[] getBytes() throws IOException {
    MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
    Visitor visitor = new Visitor(packer);
    // a message contains three fields, (type & data & nodeID)
    packer.packArrayHeader(3);
    packer.packString(this.type);
    if (this.data != null) {
        packer.packMapHeader(this.data.size());
        for (Map.Entry<String, Object> entry : this.data.entrySet()) {
            packer.packString(entry.getKey());
            visitor.visit(entry.getValue());
        }
    } else {
        packer.packNil();
    }
    packer.packString(this.nodeID);
    byte[] bytes = packer.toByteArray();
    packer.close();
    return bytes;
}
 

public A deserialize(byte[] bytes) throws IOException {
  long start = System.currentTimeMillis();
  if (null == bytes)
    return null;

  try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) {
    final long id = unpacker.unpackLong();
    final String label = unpacker.unpackString();
    final Map<String, long[]> inEdgeIdsByLabel = unpackEdgeIdsByLabel(unpacker);
    final Map<String, long[]> outEdgeIdsByLabel = unpackEdgeIdsByLabel(unpacker);
    final Map<String, Object> properties = unpackProperties(unpacker);

    A a = createElement(id, label, properties, inEdgeIdsByLabel, outEdgeIdsByLabel);

    deserializedCount++;
    deserializationTimeSpentMillis += System.currentTimeMillis() - start;
    if (deserializedCount % 100000 == 0) {
      float avgDeserializationTime = deserializationTimeSpentMillis / (float) deserializedCount;
      logger.debug("stats: deserialized " + deserializedCount + " vertices in total (avg time: " + avgDeserializationTime + "ms)");
    }
    return a;
  }
}
 

public byte[] serialize(A a) throws IOException {
  long start = System.currentTimeMillis();
  try (MessageBufferPacker packer = MessagePack.newDefaultBufferPacker()) {
    packer.packLong(getId(a));
    packer.packString(getLabel(a));

    packEdgeIds(packer, getEdgeIds(a, Direction.IN));
    packEdgeIds(packer, getEdgeIds(a, Direction.OUT));
    packProperties(packer, getProperties(a));

    serializedCount++;
    serializationTimeSpentMillis += System.currentTimeMillis() - start;
    if (serializedCount % 100000 == 0) {
      float avgSerializationTime = serializationTimeSpentMillis / (float) serializedCount;
      logger.debug("stats: serialized " + serializedCount + " instances in total (avg time: " + avgSerializationTime + "ms)");
    }
    return packer.toByteArray();
  }
}
 

@Override
public KeyValue next() throws IOException {
    if (mReader.next(mKey, mValue)) {
        try(MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(mKey.getBytes())) {
            int mapSize = unpacker.unpackMapHeader();
            long offset = 0;
            long timestamp = -1;
            byte[] keyBytes = EMPTY_BYTES;
            for (int i = 0; i < mapSize; i++) {
                int key = unpacker.unpackInt();
                switch (key) {
                    case KAFKA_MESSAGE_OFFSET:
                        offset = unpacker.unpackLong();
                        break;
                    case KAFKA_MESSAGE_TIMESTAMP:
                        timestamp = unpacker.unpackLong();
                        break;
                    case KAFKA_HASH_KEY:
                        int keySize = unpacker.unpackBinaryHeader();
                        keyBytes = new byte[keySize];
                        unpacker.readPayload(keyBytes);
                        break;
                }
            }
            return new KeyValue(offset, keyBytes, Arrays.copyOfRange(mValue.getBytes(), 0, mValue.getLength()), timestamp);
        }
    } else {
        return null;
    }
}
 

/**
 * Parses the {@code messages} from the stream using the given {@code schema}.
 */
public static <T> List<T> parseListFrom(InputStream in, Schema<T> schema, boolean numeric) throws IOException
{

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(in);

    try
    {
        return parseListFrom(unpacker, schema, numeric);
    }
    finally
    {
        unpacker.close();
    }
}
 

/**
 * Parses the {@code messages} from the stream using the given {@code schema}.
 */
public static <T> List<T> parseListFrom(MessageBufferInput in, Schema<T> schema, boolean numeric) throws IOException
{

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(in);

    try
    {
        return parseListFrom(unpacker, schema, numeric);
    }
    finally
    {
        unpacker.close();
    }
}
 

/**
 * Serializes the {@code message} into an {@link OutputStream} using the given {@code schema}.
 */
public static <T> void writeTo(OutputStream out, T message, Schema<T> schema, boolean numeric) throws IOException
{

    MessagePacker packer = MessagePack.newDefaultPacker(out);

    try
    {
        writeTo(packer, message, schema, numeric);
    }
    finally
    {
        packer.flush();
    }
}
 

/**
 * Merges the {@code message} from the {@link InputStream} using the given {@code schema}.
 */
public static <T> void mergeFrom(InputStream in, T message, Schema<T> schema, boolean numeric) throws IOException
{

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(in);

    try
    {
        mergeFrom(unpacker, message, schema, numeric);
    }
    finally
    {
        unpacker.close();
    }
}
 

/**
 * Merges the {@code message} from the {@link MessageBufferInput} using the given {@code schema}.
 */
public static <T> void mergeFrom(MessageBufferInput in, T message, Schema<T> schema, boolean numeric) throws IOException
{

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(in);

    try
    {
        mergeFrom(unpacker, message, schema, numeric);
    }
    finally
    {
        unpacker.close();
    }
}
 

/**
 * Create corresponding InfluxDBException from the message pack error body.
 * @param messagePackErrorBody
 *          the error body if any
 * @return the Exception
 */
public static InfluxDBException buildExceptionForErrorState(final InputStream messagePackErrorBody) {
  try {
    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(messagePackErrorBody);
    ImmutableMapValue mapVal = (ImmutableMapValue) unpacker.unpackValue();
    return InfluxDBException.buildExceptionFromErrorMessage(
        mapVal.map().get(new ImmutableStringValueImpl("error")).toString());
  } catch (Exception e) {
    return new InfluxDBException(e);
  }
}
 

@Override
public void write(KeyValue keyValue) throws IOException {
    byte[] kafkaKey = keyValue.hasKafkaKey() ? keyValue.getKafkaKey() : new byte[0];
    long timestamp = keyValue.getTimestamp();
    final int timestampLength = (keyValue.hasTimestamp()) ? 10 : 0;
    // output size estimate
    // 1 - map header
    // 1 - message pack key
    // 9 - max kafka offset
    // 1 - message pack key
    // 9 - kafka timestamp
    // 1 - message pack key
    // 5 - max (sane) kafka key size
    // N - size of kafka key
    // = 27 + N
    ByteArrayOutputStream out = new ByteArrayOutputStream(17 + timestampLength + kafkaKey.length);
    MessagePacker packer = MessagePack.newDefaultPacker(out)
            .packMapHeader(numberOfFieldsMappedInHeader(keyValue))
            .packInt(KAFKA_MESSAGE_OFFSET)
            .packLong(keyValue.getOffset());

    if (keyValue.hasTimestamp())
        packer.packInt(KAFKA_MESSAGE_TIMESTAMP)
                .packLong(timestamp);

    if (keyValue.hasKafkaKey())
        packer.packInt(KAFKA_HASH_KEY)
                .packBinaryHeader(kafkaKey.length)
                .writePayload(kafkaKey);

    packer.close();
    byte[] outBytes = out.toByteArray();
    this.mKey.set(outBytes, 0, outBytes.length);
    this.mValue.set(keyValue.getValue(), 0, keyValue.getValue().length);
    this.mWriter.append(this.mKey, this.mValue);
}
 

@Test(timeout = 60000)
public void testPackSpans() throws Exception {
  Random rand = new Random(123);
  byte[] arr = new byte[16384];
  ByteBuffer bb = ByteBuffer.wrap(arr);
  bb.limit(bb.capacity());
  PackedBuffer buf = new PackedBuffer(bb);
  final int NUM_TEST_SPANS = 5;
  Span[] spans = new Span[NUM_TEST_SPANS];
  for (int i = 0; i < NUM_TEST_SPANS; i++) {
    spans[i] = TestUtil.randomSpan(rand);
  }
  for (int i = 0; i < NUM_TEST_SPANS; i++) {
    buf.writeSpan(spans[i]);
  }
  LOG.info("wrote " + buf.toHexString());
  MessagePack msgpack = new MessagePack(PackedBuffer.MSGPACK_CONF);
  MessageUnpacker unpacker = msgpack.newUnpacker(arr, 0, bb.position());
  Span[] respans = new Span[NUM_TEST_SPANS];
  for (int i = 0; i < NUM_TEST_SPANS; i++) {
    respans[i] = PackedBuffer.readSpan(unpacker);
  }
  for (int i = 0; i < NUM_TEST_SPANS; i++) {
    Assert.assertEquals("Failed to read back span " + i,
        spans[i].toJson(), respans[i].toJson());
  }
}
 

/**
 * 转储常量数据
 * 
 * @return 常量数据,不支持的时候返回空
 */
public final byte[] dumpConst(HashMap<String, Object> data) throws ConvException, IOException {
    MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();

    try {
        writeData(packer, data);
    } catch (IOException e) {
        this.logErrorMessage("MessagePacker write failed.");
        e.printStackTrace();
    }

    return packer.toByteArray();
}
 

/**
 * only deserialize the part we're keeping in memory, used during startup when initializing from disk
 */
public ElementRef deserializeRef(byte[] bytes) throws IOException {
  try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes)) {
    long id = unpacker.unpackLong();
    String label = unpacker.unpackString();

    Map<String, long[]> inEdgeIdsByLabel = null;
    Map<String, long[]> outEdgeIdsByLabel = null;
    if (elementRefRequiresAdjacentElements()) {
      inEdgeIdsByLabel = unpackEdgeIdsByLabel(unpacker);
      outEdgeIdsByLabel = unpackEdgeIdsByLabel(unpacker);
    }
    return createElementRef(id, label, inEdgeIdsByLabel, outEdgeIdsByLabel);
  }
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString());

                MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
                packer.packMapHeader(1)
                        .packString("ack").packString(UUID.randomUUID().toString())
                        .close();

                // Close the input stream after closing the output stream to avoid closing a socket too early
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertEquals(exception.getClass(), TimeoutException.class);
}
 

@Override
public void serialize( Options options, OutputStream output, @Optional Object object )
{
    try( MessagePacker packer = MessagePack.newDefaultPacker( output ) )
    {
        Value value = doSerialize( options, object, true );
        packer.packValue( value );
        packer.flush();
    }
    catch( IOException ex )
    {
        throw new SerializationException( "Unable to serialize " + object, ex );
    }
}