Java Code Examples for io.netty.buffer.CompositeByteBuf#addComponents()

/**
 * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal
 * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope.
 */
public static ByteBuf toByteBuf(final BytesReference reference) {
    if (reference.length() == 0) {
        return Unpooled.EMPTY_BUFFER;
    }
    if (reference instanceof ByteBufBytesReference) {
        return ((ByteBufBytesReference) reference).toByteBuf();
    } else {
        final BytesRefIterator iterator = reference.iterator();
        // usually we have one, two, or three components from the header, the message, and a buffer
        final List<ByteBuf> buffers = new ArrayList<>(3);
        try {
            BytesRef slice;
            while ((slice = iterator.next()) != null) {
                buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
            }
            final CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
            composite.addComponents(true, buffers);
            return composite;
        } catch (IOException ex) {
            throw new AssertionError("no IO happens here", ex);
        }
    }
}
 

@Override
public List<Object> encode(ByteBufAllocator alloc) {
    ByteBuf meta = metadata(alloc);

    ByteBuf head = alloc.buffer(FDFS_HEAD_LEN);
    head.writeLong(meta.readableBytes() + size);
    head.writeByte(cmd());
    head.writeByte(ERRNO_OK);

    CompositeByteBuf cbb = alloc.compositeBuffer();
    cbb.addComponents(head, meta);
    cbb.writerIndex(head.readableBytes() + meta.readableBytes());

    List<Object> requests = new LinkedList<>();
    requests.add(cbb);
    requests.add(content);
    return requests;
}
 

private ByteBuf assembleFrameWithMetadata(ByteBuf frame, int streamId, ByteBuf header) {
  ByteBuf metadata;
  CompositeByteBuf cm = removeMetadata(streamId);

  ByteBuf decodedMetadata = PayloadFrameCodec.metadata(frame);
  if (decodedMetadata != null) {
    if (cm != null) {
      metadata = cm.addComponents(true, decodedMetadata.retain());
    } else {
      metadata = PayloadFrameCodec.metadata(frame).retain();
    }
  } else {
    metadata = cm;
  }

  ByteBuf data = assembleData(frame, streamId);

  return FragmentationCodec.encode(allocator, header, metadata, data);
}
 

@Override
public List<Object> encode(ByteBufAllocator alloc) {
    ByteBuf meta = metadata(alloc);

    ByteBuf head = alloc.buffer(FDFS_HEAD_LEN);
    head.writeLong(meta.readableBytes() + size);
    head.writeByte(cmd());
    head.writeByte(ERRNO_OK);

    CompositeByteBuf cbb = alloc.compositeBuffer();
    cbb.addComponents(head, meta);
    cbb.writerIndex(head.readableBytes() + meta.readableBytes());

    List<Object> requests = new LinkedList<>();
    requests.add(cbb);
    requests.add(content);
    return requests;
}
 

/**
 * Encode a new sub-metadata information into a composite metadata {@link CompositeByteBuf
 * buffer}, first verifying if the passed {@link String} matches a {@link WellKnownMimeType} (in
 * which case it will be encoded in a compressed fashion using the mime id of that type).
 *
 * <p>Prefer using {@link #encodeAndAddMetadata(CompositeByteBuf, ByteBufAllocator, String,
 * ByteBuf)} if you already know that the mime type is not a {@link WellKnownMimeType}.
 *
 * @param compositeMetaData the buffer that will hold all composite metadata information.
 * @param allocator the {@link ByteBufAllocator} to use to create intermediate buffers as needed.
 * @param mimeType the mime type to encode, as a {@link String}. well known mime types are
 *     compressed.
 * @param metadata the metadata value to encode.
 * @see #encodeAndAddMetadata(CompositeByteBuf, ByteBufAllocator, WellKnownMimeType, ByteBuf)
 */
// see #encodeMetadataHeader(ByteBufAllocator, String, int)
public static void encodeAndAddMetadataWithCompression(
    CompositeByteBuf compositeMetaData,
    ByteBufAllocator allocator,
    String mimeType,
    ByteBuf metadata) {
  WellKnownMimeType wkn = WellKnownMimeType.fromString(mimeType);
  if (wkn == WellKnownMimeType.UNPARSEABLE_MIME_TYPE) {
    compositeMetaData.addComponents(
        true, encodeMetadataHeader(allocator, mimeType, metadata.readableBytes()), metadata);
  } else {
    compositeMetaData.addComponents(
        true,
        encodeMetadataHeader(allocator, wkn.getIdentifier(), metadata.readableBytes()),
        metadata);
  }
}
 

/**
 * Encode a new sub-metadata information into a composite metadata {@link CompositeByteBuf
 * buffer}.
 *
 * @param compositeMetaData the buffer that will hold all composite metadata information.
 * @param allocator the {@link ByteBufAllocator} to use to create intermediate buffers as needed.
 * @param knownMimeType the {@link WellKnownMimeType} to encode.
 * @param metadata the metadata value to encode.
 */
// see #encodeMetadataHeader(ByteBufAllocator, byte, int)
public static void encodeAndAddMetadata(
    CompositeByteBuf compositeMetaData,
    ByteBufAllocator allocator,
    WellKnownMimeType knownMimeType,
    ByteBuf metadata) {
  compositeMetaData.addComponents(
      true,
      encodeMetadataHeader(allocator, knownMimeType.getIdentifier(), metadata.readableBytes()),
      metadata);
}
 

/**
 * Fill the buffer of the current chunk with the data from the given {@link ByteBuf}.
 * @param channelReadBuf the {@link ByteBuf} from which to read data.
 * @return the number of bytes transferred in this operation.
 */
int fillFrom(ByteBuf channelReadBuf) {
  int toWrite;
  if (buf == null) {
    // If current buf is null, then only read the up to routerMaxPutChunkSizeBytes.
    toWrite = Math.min(channelReadBuf.readableBytes(), routerConfig.routerMaxPutChunkSizeBytes);
    buf = channelReadBuf.readRetainedSlice(toWrite);
  } else {
    int remainingSize = routerConfig.routerMaxPutChunkSizeBytes - buf.readableBytes();
    toWrite = Math.min(channelReadBuf.readableBytes(), remainingSize);
    ByteBuf remainingSlice = channelReadBuf.readRetainedSlice(toWrite);
    // buf already has some bytes
    if (buf instanceof CompositeByteBuf) {
      // Buf is already a CompositeByteBuf, then just add the slice from
      ((CompositeByteBuf) buf).addComponent(true, remainingSlice);
    } else {
      int maxComponents = routerConfig.routerMaxPutChunkSizeBytes;
      CompositeByteBuf composite = buf.isDirect() ? buf.alloc().compositeDirectBuffer(maxComponents)
          : buf.alloc().compositeHeapBuffer(maxComponents);
      composite.addComponents(true, buf, remainingSlice);
      buf = composite;
    }
  }
  if (buf.readableBytes() == routerConfig.routerMaxPutChunkSizeBytes) {
    onFillComplete(true);
    updateChunkFillerWaitTimeMetrics();
  }
  return toWrite;
}
 

private ByteBuf assembleData(ByteBuf frame, int streamId) {
  ByteBuf data;
  CompositeByteBuf cd = removeData(streamId);
  if (cd != null) {
    cd.addComponents(true, PayloadFrameCodec.data(frame).retain());
    data = cd;
  } else {
    data = Unpooled.EMPTY_BUFFER;
  }

  return data;
}
 

/**
 * Encode a new sub-metadata information into a composite metadata {@link CompositeByteBuf
 * buffer}.
 *
 * @param compositeMetaData the buffer that will hold all composite metadata information.
 * @param allocator the {@link ByteBufAllocator} to use to create intermediate buffers as needed.
 * @param unknownCompressedMimeType the id of the {@link
 *     WellKnownMimeType#UNKNOWN_RESERVED_MIME_TYPE} to encode.
 * @param metadata the metadata value to encode.
 */
// see #encodeMetadataHeader(ByteBufAllocator, byte, int)
static void encodeAndAddMetadata(
    CompositeByteBuf compositeMetaData,
    ByteBufAllocator allocator,
    byte unknownCompressedMimeType,
    ByteBuf metadata) {
  compositeMetaData.addComponents(
      true,
      encodeMetadataHeader(allocator, unknownCompressedMimeType, metadata.readableBytes()),
      metadata);
}
 

static ByteBufJoiner wrapped() {
    return (parts) -> {
        int size = parts.size();

        switch (size) {
            case 0:
                throw new IllegalStateException("No buffer available");
            case 1:
                try {
                    return parts.get(0);
                } finally {
                    parts.clear();
                }
            default:
                CompositeByteBuf composite = null;

                try {
                    composite = parts.get(0).alloc().compositeBuffer(size);
                    // Auto-releasing failed parts
                    return composite.addComponents(true, parts);
                } catch (Throwable e) {
                    if (composite == null) {
                        // Alloc failed, release parts.
                        for (ByteBuf part : parts) {
                            ReferenceCountUtil.safeRelease(part);
                        }
                    } else {
                        // Also release success parts.
                        composite.release();
                    }
                    throw e;
                } finally {
                    parts.clear();
                }
        }
    };
}
 

private List<ByteBuf> readNextEntriesFromLedger(long start, long maxNumberEntries) throws IOException {
    long end = Math.min(start + maxNumberEntries - 1, ledger.getLastAddConfirmed());
    try (LedgerEntries ledgerEntriesOnce = ledger.readAsync(start, end).get()) {
        log.debug("read ledger entries. start: {}, end: {}", start, end);

        List<ByteBuf> entries = Lists.newLinkedList();

        Iterator<LedgerEntry> iterator = ledgerEntriesOnce.iterator();
        while (iterator.hasNext()) {
            LedgerEntry entry = iterator.next();
            ByteBuf buf = entry.getEntryBuffer().retain();
            int entryLength = buf.readableBytes();
            long entryId = entry.getEntryId();

            CompositeByteBuf entryBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer(2);
            ByteBuf entryHeaderBuf = PulsarByteBufAllocator.DEFAULT.buffer(ENTRY_HEADER_SIZE, ENTRY_HEADER_SIZE);

            entryHeaderBuf.writeInt(entryLength).writeLong(entryId);
            entryBuf.addComponents(true, entryHeaderBuf, buf);

            entries.add(entryBuf);
        }
        return entries;
    } catch (InterruptedException | ExecutionException e) {
        log.error("Exception when get CompletableFuture<LedgerEntries>. ", e);
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new IOException(e);
    }
}
 

/**
 * Returns a {@link CompositeByteBuf} that contains the same data as {@code sourceBuffer}, but with
 * SMTP dot-stuffing applied, and (if {@code} appendCRLF is true) a CRLF appended.
 *
 * <p>If dot-stuffing is not required, and {@code appendCRLF} is false, {@code sourceBuffer} is
 * returned. In all other cases, {@code allocator} will be used to create a new {@code ByteBuf}
 * with a {@code refCnt} of one.
 *
 * <p>The {@code previousBytes} parameter is used to maintain dot-stuffing across a series
 * of buffers. Pass the last two bytes of a previous buffer here to ensure an initial dot
 * will be escaped if necessary. Passing null indicates this is the first or only buffer
 * for this message.
 *
 * @param allocator the {@code ByteBufAllocator} to use for new {@code ByteBuf}s
 * @param sourceBuffer the source message data
 * @param previousBytes the previous two bytes of the message, or null
 * @param termination whether to append CRLF to the end of the returned buffer
 */
public static ByteBuf createDotStuffedBuffer(ByteBufAllocator allocator, ByteBuf sourceBuffer, byte[] previousBytes, MessageTermination termination) {
  int dotIndex = findDotAtBeginningOfLine(sourceBuffer, 0, normalisePreviousBytes(previousBytes));

  try {
    if (dotIndex == -1) {
      if (termination == MessageTermination.ADD_CRLF) {
        return allocator.compositeBuffer(2).addComponents(true, sourceBuffer.retainedSlice(), CR_LF_BUFFER.slice());
      } else {
        return sourceBuffer.retainedSlice();
      }
    }

    // Build a CompositeByteBuf to avoid copying
    CompositeByteBuf compositeByteBuf = allocator.compositeBuffer();
    compositeByteBuf.addComponents(true, sourceBuffer.retainedSlice(0, dotIndex), DOT_DOT_BUFFER.slice());

    int nextDotIndex;
    while ((nextDotIndex = findDotAtBeginningOfLine(sourceBuffer, dotIndex + 1, NOT_CR_LF)) != -1) {
      compositeByteBuf.addComponents(true, sourceBuffer.retainedSlice(dotIndex + 1, nextDotIndex - dotIndex - 1), DOT_DOT_BUFFER.slice());
      dotIndex = nextDotIndex;
    }

    compositeByteBuf.addComponent(true, sourceBuffer.retainedSlice(dotIndex + 1, sourceBuffer.readableBytes() - dotIndex - 1));

    if (termination == MessageTermination.ADD_CRLF) {
      compositeByteBuf.addComponent(true, CR_LF_BUFFER.slice());
    }

    return compositeByteBuf;
  } finally {
    sourceBuffer.release();
  }
}
 

private static ByteBuf retainedMerge(ByteBufAllocator allocator, List<ByteBuf> parts) {
    int i;
    int successSentinel = 0;
    int size = parts.size();
    CompositeByteBuf byteBuf = allocator.compositeBuffer(size);

    try {
        for (i = 0; i < size; ++i) {
            parts.get(i).retain();
            successSentinel = i + 1;
        }

        // Auto-releasing failed Buffer if addComponents called.
        return byteBuf.addComponents(true, parts);
    } catch (Throwable e) {
        // Also release components which append succeed.
        ReferenceCountUtil.safeRelease(byteBuf);

        if (successSentinel < size) {
            // Retains failed, even not call addComponents.
            // So release all retained buffers.
            // Of course, this still does not solve call-stack
            // overflow when calling addComponents.
            for (i = 0; i < successSentinel; ++i) {
                ReferenceCountUtil.safeRelease(parts.get(i));
            }
        }

        throw e;
    }
}
 

static ByteBufJoiner wrapped() {
    return (parts) -> {
        int size = parts.size();

        switch (size) {
            case 0:
                throw new IllegalStateException("No buffer available");
            case 1:
                try {
                    return parts.get(0);
                } finally {
                    parts.clear();
                }
            default:
                CompositeByteBuf composite = null;

                try {
                    composite = parts.get(0).alloc().compositeBuffer(size);
                    // Auto-releasing failed parts
                    return composite.addComponents(true, parts);
                } catch (Throwable e) {
                    if (composite == null) {
                        // Alloc failed, release parts.
                        for (ByteBuf part : parts) {
                            ReferenceCountUtil.safeRelease(part);
                        }
                    } else {
                        // Also release success parts.
                        composite.release();
                    }
                    throw e;
                } finally {
                    parts.clear();
                }
        }
    };
}
 

private static ByteBuf retainedMerge(ByteBufAllocator allocator, List<ByteBuf> parts) {
    int i;
    int successSentinel = 0;
    int size = parts.size();
    CompositeByteBuf byteBuf = allocator.compositeBuffer(size);

    try {
        for (i = 0; i < size; ++i) {
            parts.get(i).retain();
            successSentinel = i + 1;
        }

        // Auto-releasing failed Buffer if addComponents called.
        return byteBuf.addComponents(true, parts);
    } catch (Throwable e) {
        // Also release components which append succeed.
        ReferenceCountUtil.safeRelease(byteBuf);

        if (successSentinel < size) {
            // Retains failed, even not call addComponents.
            // So release all retained buffers.
            // Of course, this still does not solve call-stack
            // overflow when calling addComponents.
            for (i = 0; i < successSentinel; ++i) {
                ReferenceCountUtil.safeRelease(parts.get(i));
            }
        }

        throw e;
    }
}
 

void handleFollowsFlag(ByteBuf frame, int streamId, FrameType frameType) {
  ByteBuf header = getHeader(streamId);
  if (header == null) {
    header = frame.copy(frame.readerIndex(), FrameHeaderCodec.size());

    if (frameType == FrameType.REQUEST_CHANNEL || frameType == FrameType.REQUEST_STREAM) {
      long i = RequestChannelFrameCodec.initialRequestN(frame);
      header.writeInt(i > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) i);
    }
    putHeader(streamId, header);
  }

  if (FrameHeaderCodec.hasMetadata(frame)) {
    CompositeByteBuf metadata = getMetadata(streamId);
    switch (frameType) {
      case REQUEST_FNF:
        metadata.addComponents(true, RequestFireAndForgetFrameCodec.metadata(frame).retain());
        break;
      case REQUEST_STREAM:
        metadata.addComponents(true, RequestStreamFrameCodec.metadata(frame).retain());
        break;
      case REQUEST_RESPONSE:
        metadata.addComponents(true, RequestResponseFrameCodec.metadata(frame).retain());
        break;
      case REQUEST_CHANNEL:
        metadata.addComponents(true, RequestChannelFrameCodec.metadata(frame).retain());
        break;
        // Payload and synthetic types
      case PAYLOAD:
      case NEXT:
      case NEXT_COMPLETE:
      case COMPLETE:
        metadata.addComponents(true, PayloadFrameCodec.metadata(frame).retain());
        break;
      default:
        throw new IllegalStateException("unsupported fragment type");
    }
  }

  ByteBuf data;
  switch (frameType) {
    case REQUEST_FNF:
      data = RequestFireAndForgetFrameCodec.data(frame).retain();
      break;
    case REQUEST_STREAM:
      data = RequestStreamFrameCodec.data(frame).retain();
      break;
    case REQUEST_RESPONSE:
      data = RequestResponseFrameCodec.data(frame).retain();
      break;
    case REQUEST_CHANNEL:
      data = RequestChannelFrameCodec.data(frame).retain();
      break;
      // Payload and synthetic types
    case PAYLOAD:
    case NEXT:
    case NEXT_COMPLETE:
    case COMPLETE:
      data = PayloadFrameCodec.data(frame).retain();
      break;
    default:
      throw new IllegalStateException("unsupported fragment type");
  }

  getData(streamId).addComponents(true, data);
  frame.release();
}
 

public static @NotNull ByteBuf writeArgs(
    @NotNull ByteBufAllocator allocator,
    @NotNull ByteBuf header,
    @NotNull List<ByteBuf> args
) {
    int writableBytes = TFrame.MAX_FRAME_PAYLOAD_LENGTH - header.readableBytes();
    List<ByteBuf> allocatedBufs = new ArrayList<>(7);
    List<ByteBuf> bufs = new ArrayList<>(7);
    bufs.add(header);

    boolean release = true;
    try {
        while (!args.isEmpty()) {
            ByteBuf arg = args.get(0);
            int len = writeArg(allocator, arg, writableBytes, bufs, allocatedBufs);
            writableBytes -= len;
            if (writableBytes <= TFrame.FRAME_SIZE_LENGTH) {
                break;
            }

            if (arg.readableBytes() == 0) {
                args.remove(0);
            }
        }

        CompositeByteBuf comp = allocator.compositeBuffer();
        comp.addComponents(bufs);
        comp.writerIndex(TFrame.MAX_FRAME_PAYLOAD_LENGTH - writableBytes);
        release = false;
        return comp;
    } finally {
        if (release) {
            for (ByteBuf buf : allocatedBufs) {
                if (buf != null) {
                    try {
                        buf.release();
                    } catch (Exception e) {
                        logger.warn("Failed to release", e);
                    }
                }
            }
        }
    }

}
 

/**
 * @param args
 */
public static void main(String[] args)
{
    byte[] bs1 = new byte[]
    { 1, 10, 11, 12 };
    byte[] bs2 = new byte[]
    { 2, 2, 2, 2 };
    byte[] bs3 = new byte[]
    { 3, 3, 3, 3 };
    byte[] bs4 = new byte[]
    { 4, 4, 4, 4 };
    byte[] bs5 = new byte[]
    { 5, 5, 5, 5 };
    byte[] bs6 = new byte[]
    { 6, 6, 6, 6 };

    ByteBuffer buffer1 = ByteBuffer.allocate(1024);
    buffer1.put(bs1);
    buffer1.flip();

    ByteBuf buf1 = Unpooled.copiedBuffer(buffer1);// .copiedBuffer(bs1);

    buffer1.put(bs3);

    ByteBuf buf2 = Unpooled.copiedBuffer(bs2);
    ByteBuf buf3 = Unpooled.copiedBuffer(bs3);
    ByteBuf buf4 = Unpooled.copiedBuffer(bs4);
    ByteBuf buf5 = Unpooled.copiedBuffer(bs5);
    ByteBuf buf6 = Unpooled.copiedBuffer(bs6);

    CompositeByteBuf cb = Unpooled.compositeBuffer();
    cb.addComponents(buf1, buf2, buf3);

    byte dd = cb.getByte(0);

    CompositeByteBuf cb2 = Unpooled.compositeBuffer();
    cb.addComponents(buf4, buf5, buf6);

    // cb.c
    // cb2.writerIndex(128 * 1024);

    cb.addComponent(cb2);

    Long number = cb2.readLong(); // causes IllegalBufferAccessException
                                  // here!

}
 

/**
 * Encode a new sub-metadata information into a composite metadata {@link CompositeByteBuf
 * buffer}, without checking if the {@link String} can be matched with a well known compressable
 * mime type. Prefer using this method and {@link #encodeAndAddMetadata(CompositeByteBuf,
 * ByteBufAllocator, WellKnownMimeType, ByteBuf)} if you know in advance whether or not the mime
 * is well known. Otherwise use {@link #encodeAndAddMetadataWithCompression(CompositeByteBuf,
 * ByteBufAllocator, String, ByteBuf)}
 *
 * @param compositeMetaData the buffer that will hold all composite metadata information.
 * @param allocator the {@link ByteBufAllocator} to use to create intermediate buffers as needed.
 * @param customMimeType the custom mime type to encode.
 * @param metadata the metadata value to encode.
 */
// see #encodeMetadataHeader(ByteBufAllocator, String, int)
public static void encodeAndAddMetadata(
    CompositeByteBuf compositeMetaData,
    ByteBufAllocator allocator,
    String customMimeType,
    ByteBuf metadata) {
  compositeMetaData.addComponents(
      true, encodeMetadataHeader(allocator, customMimeType, metadata.readableBytes()), metadata);
}