org.apache.thrift.transport.TIOStreamTransport Java Examples

The following examples show how to use org.apache.thrift.transport.TIOStreamTransport. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleThriftLogger.java    From singer with Apache License 2.0 6 votes vote down vote up
/**
 * Simple implementation of log file rotation.
 * @throws java.io.IOException
 */
public void rotate() throws IOException {
  close();

  int i = 0;
  while (new File(String.format("%s.%d", fileName, ++i)).exists()) {
    ;
  }

  for (int j = i - 1; j >= 1; --j) {
    FileUtils.moveFile(
        new File(String.format("%s.%d", fileName, j)),
        new File(String.format("%s.%d", fileName, j + 1)));
  }
  FileUtils.moveFile(new File(fileName), new File(fileName + ".1"));
  bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, true));
  transport = new ByteOffsetTFramedTransport(new TIOStreamTransport(bufferedOutputStream));
  protocol = new TBinaryProtocol(transport);
}
 
Example #2
Source File: TTextProtocolTest.java    From armeria with Apache License 2.0 6 votes vote down vote up
@Test(expected = TException.class)
public void rpcNoMethod() throws Exception {
    final String request =
            "{\n" +
            "  \"type\" : \"CALL\",\n" +
            "  \"args\" : {\n" +
            "    \"methodArg1\" : \"foo1\",\n" +
            "    \"methodArg2\" : 200,\n" +
            "    \"details\" : {\n" +
            "      \"detailsArg1\" : \"foo2\",\n" +
            "      \"detailsArg2\" : 100\n" +
            "    }\n" +
            "  }\n" +
            '}';
    final TTextProtocol prot = new TTextProtocol(
            new TIOStreamTransport(new ByteArrayInputStream(request.getBytes())));
    prot.readMessageBegin();
}
 
Example #3
Source File: TestThriftToParquetFileWriter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private <T extends TBase<?,?>> Path createFile(T... tObjs) throws IOException, InterruptedException, TException  {
  final Path fileToCreate = new Path("target/test/TestThriftToParquetFileWriter/"+tObjs[0].getClass()+".parquet");
  LOG.info("File created: {}", fileToCreate.toString());
  Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) {
    fs.delete(fileToCreate, true);

  }
  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, (Class<? extends TBase<?, ?>>) tObjs[0].getClass());

  for(T tObj:tObjs) {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

    tObj.write(protocol);

    w.write(new BytesWritable(baos.toByteArray()));
  }
  w.close();

  return fileToCreate;
}
 
Example #4
Source File: TTextProtocolTest.java    From armeria with Apache License 2.0 6 votes vote down vote up
/**
 * Read in (deserialize) a thrift message in TTextProtocol format
 * from a file on disk, then serialize it back out to a string.
 * Finally, deserialize that string and compare to the original
 * message.
 */
@Test
public void tTextProtocolReadWriteTest() throws Exception {
    // Deserialize the file contents into a thrift message.
    final ByteArrayInputStream bais1 = new ByteArrayInputStream(testData.getBytes());

    final TTextProtocolTestMsg msg1 = new TTextProtocolTestMsg();
    msg1.read(new TTextProtocol(new TIOStreamTransport(bais1)));

    assertThat(msg1).isEqualTo(testMsg());

    // Serialize that thrift message out to a byte array
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    msg1.write(new TTextProtocol(new TIOStreamTransport(baos), true));
    final byte[] bytes = baos.toByteArray();

    // Deserialize that string back to a thrift message.
    final ByteArrayInputStream bais2 = new ByteArrayInputStream(bytes);
    final TTextProtocolTestMsg msg2 = new TTextProtocolTestMsg();
    msg2.read(new TTextProtocol(new TIOStreamTransport(bais2)));

    assertThat(msg2).isEqualTo(msg1);
}
 
Example #5
Source File: ThriftFrameEncoder.java    From floodlight_with_topoguard with Apache License 2.0 6 votes vote down vote up
@Override
protected Object encode(ChannelHandlerContext ctx, Channel channel,
                        Object message) throws Exception {
    if (message instanceof SyncMessage) {
        ChannelBuffer buf = new DynamicChannelBuffer(512);
        ChannelBufferOutputStream os = new ChannelBufferOutputStream(buf);
        TCompactProtocol thriftProtocol =
                new TCompactProtocol(new TIOStreamTransport(os));
        ((SyncMessage) message).write(thriftProtocol);

        ChannelBuffer len = ChannelBuffers.buffer(4);
        len.writeInt(buf.readableBytes());
        return ChannelBuffers.wrappedBuffer(len, buf);
    }
    return message;
}
 
Example #6
Source File: ThriftFrameDecoder.java    From floodlight_with_topoguard with Apache License 2.0 6 votes vote down vote up
@Override
protected Object decode(ChannelHandlerContext ctx,
                        Channel channel,
                        ChannelBuffer buffer) throws Exception {
    List<SyncMessage> ms = null;
    ChannelBuffer frame = null;
    while (null != (frame = (ChannelBuffer) super.decode(ctx, channel, 
                                                         buffer))) {
        if (ms == null) ms = new ArrayList<SyncMessage>();
        ChannelBufferInputStream is = new ChannelBufferInputStream(frame);
        TCompactProtocol thriftProtocol =
                new TCompactProtocol(new TIOStreamTransport(is));
        SyncMessage bsm = new SyncMessage();
        bsm.read(thriftProtocol);
        ms.add(bsm);
    }
    return ms;
}
 
Example #7
Source File: ThriftBinaryCodec.java    From attic-aurora with Apache License 2.0 6 votes vote down vote up
/**
 * Encodes a thrift object into a DEFLATE-compressed binary array.
 *
 * @param tBase Object to encode.
 * @return Deflated, encoded object.
 * @throws CodingException If the object could not be encoded.
 */
public static byte[] deflateNonNull(TBase<?, ?> tBase) throws CodingException {
  requireNonNull(tBase);

  // NOTE: Buffering is needed here for performance.
  // There are actually 2 buffers in play here - the BufferedOutputStream prevents thrift from
  // causing a call to deflate() on every encoded primitive. The DeflaterOutputStream buffer
  // allows the underlying Deflater to operate on a larger chunk at a time without stopping to
  // copy the intermediate compressed output to outBytes.
  // See http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4986239
  ByteArrayOutputStream outBytes = new ByteArrayOutputStream();
  TTransport transport = new TIOStreamTransport(
      new BufferedOutputStream(
          new DeflaterOutputStream(outBytes, new Deflater(DEFLATE_LEVEL), DEFLATER_BUFFER_SIZE),
          DEFLATER_BUFFER_SIZE));
  try {
    TProtocol protocol = PROTOCOL_FACTORY.getProtocol(transport);
    tBase.write(protocol);
    transport.close(); // calls finish() on the underlying stream, completing the compression
    return outBytes.toByteArray();
  } catch (TException e) {
    throw new CodingException("Failed to serialize: " + tBase, e);
  } finally {
    transport.close();
  }
}
 
Example #8
Source File: ThriftReader.java    From singer with Apache License 2.0 6 votes vote down vote up
public ThriftReader(
    String path,
    TBaseFactory<T> baseFactory,
    TProtocolFactory protocolFactory,
    int readBufferSize,
    int maxMessageSize) throws IOException {
  Preconditions.checkArgument(!Strings.isNullOrEmpty(path));
  Preconditions.checkNotNull(protocolFactory);

  this.byteOffsetInputStream = new ByteOffsetInputStream(
      new RandomAccessFile(path, "r"), readBufferSize);
  this.framedTransport = new TFramedTransport(new TIOStreamTransport(this
      .byteOffsetInputStream), maxMessageSize);
  this.baseFactory = Preconditions.checkNotNull(baseFactory);
  this.protocol = protocolFactory.get(this.framedTransport);
}
 
Example #9
Source File: ThriftIO.java    From beam with Apache License 2.0 5 votes vote down vote up
public void write(T element) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  try {
    element.write(protocol);
  } catch (TException te) {
    LOG.error("Error in writing element to TProtocol: " + te);
    throw new RuntimeException(te);
  }
  this.stream.write(baos.toByteArray());
}
 
Example #10
Source File: TestCorruptScroogeRecords.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Override
protected void assertEqualsExcepted(List<org.apache.parquet.thrift.test.compat.StructWithUnionV2> expected, List<Object> found) throws Exception {
  List<StructWithUnionV2> scroogeExpected = new ArrayList<StructWithUnionV2>();
  for (org.apache.parquet.thrift.test.compat.StructWithUnionV2 tbase : expected) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    TProtocol out = new Factory().getProtocol(new TIOStreamTransport(baos));
    tbase.write(out);
    TProtocol in = new Factory().getProtocol(new TIOStreamTransport(new ByteArrayInputStream(baos.toByteArray())));
    scroogeExpected.add(StructWithUnionV2$.MODULE$.decode(in));
  }
  assertEquals(scroogeExpected, found);
 }
 
Example #11
Source File: TTextProtocolTest.java    From armeria with Apache License 2.0 5 votes vote down vote up
@Test(expected = TException.class)
public void rpcArgsNotObject() throws Exception {
    final String request =
            "{\n" +
            "  \"method\" : \"doDebug\",\n" +
            "  \"args\" : 100\n" +
            '}';
    final TTextProtocol prot = new TTextProtocol(
            new TIOStreamTransport(new ByteArrayInputStream(request.getBytes())));
    prot.readMessageBegin();
}
 
Example #12
Source File: ThriftSampleData.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private void writeObject(ObjectOutputStream out)
    throws IOException {
  try {
    this.write(new TCompactProtocol(new TIOStreamTransport(out)));
  } catch (TException var3) {
    throw new IOException(var3);
  }
}
 
Example #13
Source File: AppenderUtils.java    From singer with Apache License 2.0 5 votes vote down vote up
@Override
public void init(OutputStream os) {
  this.os = os;
  // Use the TFlushingFastFramedTransport to be compatible with singer_thrift
  // log.
  final int bufferCapacity = 10;
  framedTransport = new TFastFramedTransport(new TIOStreamTransport(os),
      bufferCapacity);
  protocol = new TBinaryProtocol(framedTransport);
}
 
Example #14
Source File: TestParquetTupleScheme.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void createFileForRead() throws Exception {
  final Path fileToCreate = new Path(parquetInputPath + "/names.parquet");

  final Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) fs.delete(fileToCreate, true);

  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, Name.class);

  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  Name n1 = new Name();
  n1.setFirst_name("Alice");
  n1.setLast_name("Practice");
  Name n2 = new Name();
  n2.setFirst_name("Bob");
  n2.setLast_name("Hope");
  Name n3 = new Name();
  n3.setFirst_name("Charlie");
  n3.setLast_name("Horse");

  n1.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n2.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n3.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 
Example #15
Source File: TestParquetToThriftReadWriteAndProjection.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private <T extends TBase<?, ?>> void shouldDoProjection(Configuration conf, T recordToWrite, T exptectedReadResult, Class<? extends TBase<?, ?>> thriftClass) throws Exception {
  final Path parquetFile = new Path("target/test/TestParquetToThriftReadWriteAndProjection/file.parquet");
  final FileSystem fs = parquetFile.getFileSystem(conf);
  if (fs.exists(parquetFile)) {
    fs.delete(parquetFile, true);
  }

  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, thriftClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  recordToWrite.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();


  final ParquetThriftInputFormat<T> parquetThriftInputFormat = new ParquetThriftInputFormat<T>();
  final Job job = new Job(conf, "read");
  job.setInputFormatClass(ParquetThriftInputFormat.class);
  ParquetThriftInputFormat.setInputPaths(job, parquetFile);
  final JobID jobID = new JobID("local", 1);
  List<InputSplit> splits = parquetThriftInputFormat.getSplits(ContextUtil.newJobContext(ContextUtil.getConfiguration(job), jobID));
  T readValue = null;
  for (InputSplit split : splits) {
    TaskAttemptContext taskAttemptContext = ContextUtil.newTaskAttemptContext(ContextUtil.getConfiguration(job), new TaskAttemptID(new TaskID(jobID, true, 1), 0));
    final RecordReader<Void, T> reader = parquetThriftInputFormat.createRecordReader(split, taskAttemptContext);
    reader.initialize(split, taskAttemptContext);
    if (reader.nextKeyValue()) {
      readValue = reader.getCurrentValue();
      LOG.info("{}", readValue);
    }
  }
  assertEquals(exptectedReadResult, readValue);

}
 
Example #16
Source File: TTextProtocolTest.java    From armeria with Apache License 2.0 5 votes vote down vote up
@Test(expected = TException.class)
public void noRpcArgs() throws Exception {
    final String request =
            "{\n" +
            "  \"method\" : \"doDebug\"\n" +
            "  \"type\" : \"CALL\",\n" +
            '}';
    final TTextProtocol prot = new TTextProtocol(
            new TIOStreamTransport(new ByteArrayInputStream(request.getBytes())));
    prot.readMessageBegin();
}
 
Example #17
Source File: HeaderTBaseSerializer.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
/**
 * Create a new HeaderTBaseSerializer. 
 */
HeaderTBaseSerializer(ResettableByteArrayOutputStream bos, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.baos = bos;
    TIOStreamTransport transport = new TIOStreamTransport(bos);
    this.protocol = protocolFactory.getProtocol(transport);
    this.locator = locator;
}
 
Example #18
Source File: TTextProtocolTest.java    From armeria with Apache License 2.0 5 votes vote down vote up
@Test
public void rpcCall_noSeqId() throws Exception {
    final String request =
            "{\n" +
            "  \"method\" : \"doDebug\",\n" +
            "  \"type\" : \"CALL\",\n" +
            "  \"args\" : {\n" +
            "    \"methodArg1\" : \"foo1\",\n" +
            "    \"methodArg2\" : 200,\n" +
            "    \"details\" : {\n" +
            "      \"detailsArg1\" : \"foo2\",\n" +
            "      \"detailsArg2\" : 100\n" +
            "    }\n" +
            "  }\n" +
            '}';

    final TTextProtocol prot = new TTextProtocol(
            new TIOStreamTransport(new ByteArrayInputStream(request.getBytes())));
    final TMessage header = prot.readMessageBegin();
    final doDebug_args args = new RpcDebugService.Processor.doDebug().getEmptyArgsInstance();
    args.read(prot);
    prot.readMessageEnd();

    assertThat(header.name).isEqualTo("doDebug");
    assertThat(header.type).isEqualTo(TMessageType.CALL);
    assertThat(header.seqid).isZero();
}
 
Example #19
Source File: SerializationHelper.java    From ECFileCache with Apache License 2.0 5 votes vote down vote up
/**
 * convert thrift object to bytes
 *
 * @param obj thrift object
 * @param <T> Class type
 * @return bytes data
 */
public static <T extends TBase<T, ?>> byte[] toBytes(T obj) {
  Validate.notNull(obj);

  ByteArrayOutputStream buffer = new ByteArrayOutputStream();
  TTransport trans = new TIOStreamTransport(buffer);
  TCompactProtocol protocol = new TCompactProtocol(trans);
  try {
    obj.write(protocol);
    return buffer.toByteArray();
  } catch (TException e) {
    throw new IllegalStateException("unexpected", e);
  }
}
 
Example #20
Source File: ThriftNativeCodec.java    From dubbox-hystrix with Apache License 2.0 5 votes vote down vote up
protected static TProtocol newProtocol(URL url, ChannelBuffer buffer) throws IOException {
    String protocol = url.getParameter(ThriftConstants.THRIFT_PROTOCOL_KEY,
                                       ThriftConstants.DEFAULT_PROTOCOL);
    if (ThriftConstants.BINARY_THRIFT_PROTOCOL.equals(protocol)) {
        return new TBinaryProtocol(new TIOStreamTransport(new ChannelBufferOutputStream(buffer)));
    }
    throw new IOException("Unsupported protocol type " + protocol);
}
 
Example #21
Source File: InternalScribeCodecTest.java    From zipkin-reporter-java with Apache License 2.0 5 votes vote down vote up
@Test
public void sendsSpansExpectedMetrics() throws Exception {
  byte[] thrift = SpanBytesEncoder.THRIFT.encode(CLIENT_SPAN);
  List<byte[]> encodedSpans = asList(thrift, thrift);

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  TBinaryProtocol prot = new TBinaryProtocol(new TIOStreamTransport(out));

  InternalScribeCodec.writeLogRequest(ScribeClient.category, encodedSpans, 1, prot);

  assertThat(InternalScribeCodec.messageSizeInBytes(ScribeClient.category, encodedSpans))
      .isEqualTo(out.size());
}
 
Example #22
Source File: ThriftUtil.java    From buck with Apache License 2.0 5 votes vote down vote up
public static void serialize(ThriftProtocol protocol, TBase<?, ?> source, OutputStream stream)
    throws ThriftException {
  try (TTransport transport = new TIOStreamTransport(stream)) {
    TProtocol thriftProtocol = getProtocolFactory(protocol).getProtocol(transport);
    try {
      source.write(thriftProtocol);
    } catch (TException e) {
      throw new ThriftException(e);
    }
  }
}
 
Example #23
Source File: ThriftCoder.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Encodes the given value of type {@code T} onto the given output stream using provided {@link
 * ThriftCoder#protocolFactory}.
 *
 * @param value {@link org.apache.thrift.TBase} to encode.
 * @param outStream stream to output encoded value to.
 * @throws IOException if writing to the {@code OutputStream} fails for some reason
 * @throws CoderException if the value could not be encoded for some reason
 */
@Override
public void encode(T value, OutputStream outStream) throws CoderException, IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  try {
    TBase<?, ?> tBase = (TBase<?, ?>) value;
    tBase.write(protocol);
  } catch (Exception te) {
    throw new CoderException("Could not write value. Error: " + te.getMessage());
  }
  outStream.write(baos.toByteArray());
}
 
Example #24
Source File: ThriftCoder.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Decodes a value of type {@code T} from the given input stream using provided {@link
 * ThriftCoder#protocolFactory}. Returns the decoded value.
 *
 * @param inStream stream of input values to be decoded
 * @throws IOException if reading from the {@code InputStream} fails for some reason
 * @throws CoderException if the value could not be decoded for some reason
 * @return {@link TBase} decoded object
 */
@Override
public T decode(InputStream inStream) throws CoderException, IOException {
  try {
    TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(inStream));
    TBase<?, ?> value = (TBase<?, ?>) type.getDeclaredConstructor().newInstance();
    value.read(protocol);
    return (T) value;
  } catch (Exception te) {
    throw new CoderException("Could not read value. Error: " + te.getMessage());
  }
}
 
Example #25
Source File: ParquetScroogeSchemeTest.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void writeParquetFile(List<TBase> recordsToWrite, Configuration conf, Path parquetFile) throws IOException, InterruptedException, org.apache.thrift.TException {
  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  Class writeClass = recordsToWrite.get(0).getClass();
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, writeClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  for (TBase recordToWrite : recordsToWrite) {
    recordToWrite.write(protocol);
  }
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 
Example #26
Source File: StructBase.java    From Firefly with Apache License 2.0 5 votes vote down vote up
@Override
public void writeToParcel(Parcel dest, int flags) {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    TTransport transport = new TIOStreamTransport(stream);
    TProtocol protocol = new TBinaryProtocol(transport);
    TypeAdapter typeAdapter = Thrift.instance.getAdapter(this.getClass());
    try {
        typeAdapter.write(this, protocol);
    } catch (TException e) {
        throw new RuntimeException(e);
    }
    dest.writeByteArray(stream.toByteArray());
}
 
Example #27
Source File: CrossflowServlet.java    From scava with Eclipse Public License 2.0 5 votes vote down vote up
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response)
    throws ServletException, IOException {

  TTransport inTransport = null;
  TTransport outTransport = null;

  try {
    response.setContentType("application/x-thrift");

    if (null != this.customHeaders) {
      for (Map.Entry<String, String> header : this.customHeaders) {
        response.addHeader(header.getKey(), header.getValue());
      }
    }
    InputStream in = request.getInputStream();
    OutputStream out = response.getOutputStream();

    TTransport transport = new TIOStreamTransport(in, out);
    inTransport = transport;
    outTransport = transport;

    TProtocol inProtocol = protocolFactory.getProtocol(inTransport);
    TProtocol outProtocol = protocolFactory.getProtocol(outTransport);

    processor.process(inProtocol, outProtocol);
    out.flush();
  } catch (TException te) {
    throw new ServletException(te);
  }
}
 
Example #28
Source File: StreamTransformer.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
private static void transform(final AsyncLogWriter writer,
                              LogRecordWithDLSN record,
                              Transformer<byte[], byte[]> replicationTransformer,
                              final CountDownLatch keepAliveLatch)
        throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord =
            new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data))
            .addEventListener(new FutureEventListener<DLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}
 
Example #29
Source File: StreamTransformer.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
private static void transform(final AsyncLogWriter writer,
                              LogRecordWithDLSN record,
                              Transformer<byte[], byte[]> replicationTransformer,
                              final CountDownLatch keepAliveLatch)
        throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord =
            new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data))
            .whenComplete(new FutureEventListener<DLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}
 
Example #30
Source File: TestParquetTBaseScheme.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void createFileForRead() throws Exception {
  final Path fileToCreate = new Path(parquetInputPath+"/names.parquet");

  final Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) fs.delete(fileToCreate, true);

  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, Name.class);

  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  Name n1 = new Name();
  n1.setFirst_name("Alice");
  n1.setLast_name("Practice");
  Name n2 = new Name();
  n2.setFirst_name("Bob");
  n2.setLast_name("Hope");
  Name n3 = new Name();
  n3.setFirst_name("Charlie");
  n3.setLast_name("Horse");

  n1.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n2.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n3.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}