Java Code Examples for org.apache.thrift.protocol.TProtocolFactory

The following examples show how to use org.apache.thrift.protocol.TProtocolFactory. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: dk-fitting   Source File: DKMLServer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 阻塞式、多线程处理
 *
 * @param args
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) {
    try {
        int port = Integer.valueOf(prop.get("dkml.port"));
        //设置传输通道,普通通道
        TServerTransport serverTransport = new TServerSocket(port);
        //使用高密度二进制协议
        TProtocolFactory proFactory = new TCompactProtocol.Factory();
        //设置处理器
        TProcessor processor = new DKML.Processor(new DKMLImpl());
        //创建服务器
        TServer server = new TThreadPoolServer(
                new TThreadPoolServer.Args(serverTransport)
                        .protocolFactory(proFactory)
                        .processor(processor)
        );

        System.out.println("Start server on port "+ port +"...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 2
Source Project: java-specialagent   Source File: ThriftTest.java    License: Apache License 2.0 6 votes vote down vote up
private void startNewThreadPoolServer() throws Exception {
  final TServerTransport transport = new TServerSocket(port);
  final TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
  // TTransportFactory transportFactory = new TFramedTransport.Factory();

  final CustomHandler customHandler = new CustomHandler();
  final TProcessor customProcessor = new CustomService.Processor<CustomService.Iface>(customHandler);

  final TThreadPoolServer.Args args = new TThreadPoolServer.Args(transport)
    .processorFactory(new TProcessorFactory(customProcessor))
    .protocolFactory(protocolFactory)
    // .transportFactory(transportFactory)
    .minWorkerThreads(5)
    // .executorService(Executors.newCachedThreadPool())
    .maxWorkerThreads(10);

  server = new TThreadPoolServer(args);
  new Thread(new Runnable() {
    @Override
    public void run() {
      server.serve();
    }
  }).start();
}
 
Example 3
Source Project: java-specialagent   Source File: ThriftITest.java    License: Apache License 2.0 6 votes vote down vote up
private static TServer startNewThreadPoolServer(final TServerTransport transport) {
  final TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();

  final CustomHandler customHandler = new CustomHandler();
  final CustomService.Processor<CustomService.Iface> customProcessor = new CustomService.Processor<CustomService.Iface>(customHandler);

  final TThreadPoolServer.Args args = new TThreadPoolServer
    .Args(transport)
    .processorFactory(new TProcessorFactory(customProcessor))
    .protocolFactory(protocolFactory)
    .minWorkerThreads(5)
    .maxWorkerThreads(10);

  final TServer server = new TThreadPoolServer(args);
  new Thread(new Runnable() {
    @Override
    public void run() {
      server.serve();
    }
  }).start();

  return server;
}
 
Example 4
Source Project: nettythrift   Source File: ThriftMessageEncoder.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void messageReceived(ChannelHandlerContext ctx, ThriftMessage message) throws Exception {
	ByteBuf buffer = message.getContent();
	logger.debug("msg.content:: {}", buffer);
	try {
		TNettyTransport transport = new TNettyTransport(ctx.channel(), buffer);
		TProtocolFactory protocolFactory = message.getProtocolFactory();
		TProtocol protocol = protocolFactory.getProtocol(transport);
		serverDef.nettyProcessor.process(ctx, protocol, protocol,
				new DefaultWriterListener(message, transport, ctx, serverDef));
	} catch (Throwable ex) {
		int refCount = buffer.refCnt();
		if (refCount > 0) {
			buffer.release(refCount);
		}
		throw ex;
	}
}
 
Example 5
Source Project: buck   Source File: ThriftUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static TProtocolFactory getProtocolFactory(ThriftProtocol protocol) {
  // TODO(ruibm): Check whether the Factories are thread safe so we can static initialize
  // them just once.
  switch (protocol) {
    case JSON:
      return new TJSONProtocol.Factory();

    case COMPACT:
      return new TCompactProtocol.Factory();

    case BINARY:
      return new TBinaryProtocol.Factory();

    default:
      throw new IllegalArgumentException(
          String.format("Unknown ThriftProtocol [%s].", protocol.toString()));
  }
}
 
Example 6
Source Project: armeria   Source File: ThriftProtocolFactories.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the {@link TProtocolFactory} for the specified {@link SerializationFormat}.
 *
 * @throws IllegalArgumentException if the specified {@link SerializationFormat} is not for Thrift
 */
public static TProtocolFactory get(SerializationFormat serializationFormat) {
    requireNonNull(serializationFormat, "serializationFormat");

    if (serializationFormat == ThriftSerializationFormats.BINARY) {
        return BINARY;
    }

    if (serializationFormat == ThriftSerializationFormats.COMPACT) {
        return COMPACT;
    }

    if (serializationFormat == ThriftSerializationFormats.JSON) {
        return JSON;
    }

    if (serializationFormat == ThriftSerializationFormats.TEXT) {
        return TEXT;
    }

    if (serializationFormat == ThriftSerializationFormats.TEXT_NAMED_ENUM) {
        return TEXT_NAMED_ENUM;
    }

    throw new IllegalArgumentException("non-Thrift serializationFormat: " + serializationFormat);
}
 
Example 7
Source Project: armeria   Source File: ThriftProtocolFactories.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the {@link SerializationFormat} for the specified {@link TProtocolFactory}.
 *
 * @throws IllegalArgumentException if the specified {@link TProtocolFactory} is not known by this class
 */
public static SerializationFormat toSerializationFormat(TProtocolFactory protoFactory) {
    requireNonNull(protoFactory, "protoFactory");

    if (protoFactory instanceof TBinaryProtocol.Factory) {
        return ThriftSerializationFormats.BINARY;
    } else if (protoFactory instanceof TCompactProtocol.Factory) {
        return ThriftSerializationFormats.COMPACT;
    } else if (protoFactory instanceof TJSONProtocol.Factory) {
        return ThriftSerializationFormats.JSON;
    } else if (protoFactory instanceof TTextProtocolFactory) {
        final TTextProtocolFactory factory = (TTextProtocolFactory) protoFactory;
        return factory.usesNamedEnums() ? ThriftSerializationFormats.TEXT_NAMED_ENUM
                                        : ThriftSerializationFormats.TEXT;
    } else {
        throw new IllegalArgumentException(
                "unsupported TProtocolFactory: " + protoFactory.getClass().getName());
    }
}
 
Example 8
Source Project: hbase   Source File: ThriftServer.java    License: Apache License 2.0 6 votes vote down vote up
protected TServer getTHsHaServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
  THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
  int queueSize = conf.getInt(TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY,
      TBoundedThreadPoolServer.DEFAULT_MAX_QUEUED_REQUESTS);
  CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(queueSize), metrics);
  int workerThread = conf.getInt(TBoundedThreadPoolServer.MAX_WORKER_THREADS_CONF_KEY,
      serverArgs.getMaxWorkerThreads());
  ExecutorService executorService = createExecutor(
      callQueue, workerThread, workerThread);
  serverArgs.executorService(executorService).processor(processor)
      .transportFactory(transportFactory).protocolFactory(protocolFactory);
  return new THsHaServer(serverArgs);
}
 
Example 9
Source Project: hbase   Source File: ThriftServer.java    License: Apache License 2.0 6 votes vote down vote up
protected TServer getTThreadedSelectorServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase ThreadedSelector Thrift server on " + inetSocketAddress.toString());
  TThreadedSelectorServer.Args serverArgs =
      new HThreadedSelectorServerArgs(serverTransport, conf);
  int queueSize = conf.getInt(TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY,
      TBoundedThreadPoolServer.DEFAULT_MAX_QUEUED_REQUESTS);
  CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(queueSize), metrics);
  int workerThreads = conf.getInt(TBoundedThreadPoolServer.MAX_WORKER_THREADS_CONF_KEY,
      serverArgs.getWorkerThreads());
  int selectorThreads = conf.getInt(THRIFT_SELECTOR_NUM, serverArgs.getSelectorThreads());
  serverArgs.selectorThreads(selectorThreads);
  ExecutorService executorService = createExecutor(
      callQueue, workerThreads, workerThreads);
  serverArgs.executorService(executorService).processor(processor)
      .transportFactory(transportFactory).protocolFactory(protocolFactory);
  return new TThreadedSelectorServer(serverArgs);
}
 
Example 10
Source Project: hbase   Source File: ThriftServer.java    License: Apache License 2.0 6 votes vote down vote up
protected TServer getTThreadPoolServer(TProtocolFactory protocolFactory, TProcessor processor,
    TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws Exception {
  LOG.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
  // Thrift's implementation uses '0' as a placeholder for 'use the default.'
  int backlog = conf.getInt(BACKLOG_CONF_KEY, BACKLOG_CONF_DEAFULT);
  int readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
      THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
  TServerTransport serverTransport = new TServerSocket(
      new TServerSocket.ServerSocketTransportArgs().
          bindAddr(inetSocketAddress).backlog(backlog).
          clientTimeout(readTimeout));

  TBoundedThreadPoolServer.Args serverArgs =
      new TBoundedThreadPoolServer.Args(serverTransport, conf);
  serverArgs.processor(processor).transportFactory(transportFactory)
      .protocolFactory(protocolFactory);
  return new TBoundedThreadPoolServer(serverArgs, metrics);
}
 
Example 11
Source Project: parquet-mr   Source File: TestThriftToParquetFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
private <T extends TBase<?,?>> Path createFile(T... tObjs) throws IOException, InterruptedException, TException  {
  final Path fileToCreate = new Path("target/test/TestThriftToParquetFileWriter/"+tObjs[0].getClass()+".parquet");
  LOG.info("File created: {}", fileToCreate.toString());
  Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) {
    fs.delete(fileToCreate, true);

  }
  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, (Class<? extends TBase<?, ?>>) tObjs[0].getClass());

  for(T tObj:tObjs) {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

    tObj.write(protocol);

    w.write(new BytesWritable(baos.toByteArray()));
  }
  w.close();

  return fileToCreate;
}
 
Example 12
Source Project: secor   Source File: ThriftMessageParser.java    License: Apache License 2.0 6 votes vote down vote up
public ThriftMessageParser(SecorConfig config)
        throws InstantiationException, IllegalAccessException,
        ClassNotFoundException {
    super(config);
    TProtocolFactory protocolFactory = null;
    String protocolName = mConfig.getThriftProtocolClass();
    
    if (StringUtils.isNotEmpty(protocolName)) {
        String factoryClassName = protocolName.concat("$Factory");
        protocolFactory = ((Class<? extends TProtocolFactory>) Class.forName(factoryClassName)).newInstance();
    } else
        protocolFactory = new TBinaryProtocol.Factory();
    
    mDeserializer = new TDeserializer(protocolFactory);
    mThriftPath = new ThriftPath(mConfig.getMessageTimestampName(),(short) mConfig.getMessageTimestampId());
    mTimestampType = mConfig.getMessageTimestampType();
}
 
Example 13
Source Project: thrift-mock   Source File: ServerConfig.java    License: Apache License 2.0 5 votes vote down vote up
public ServerConfig(int port, TProtocolFactory tProtocolFactory,
                    int minWorkerThread, int maxWorkerThread) {
  this.port = port;
  this.tProtocolFactory = tProtocolFactory;
  this.minWorkerThread = minWorkerThread;
  this.maxWorkerThread = maxWorkerThread;
}
 
Example 14
Source Project: dk-fitting   Source File: DKGraphxServer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 阻塞式、多线程处理
 *
 * @param args
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) {
    try {
        int port = Integer.valueOf(prop.get("dkgraphx.port"));
        //设置传输通道,普通通道
        TServerTransport serverTransport = new TServerSocket(port);

        //使用高密度二进制协议
        TProtocolFactory proFactory = new TCompactProtocol.Factory();

        //设置处理器
        TProcessor processor = new DKGraphx.Processor(new DKGraphxImpl());

        //创建服务器
        TServer server = new TThreadPoolServer(
                new TThreadPoolServer.Args(serverTransport)
                        .protocolFactory(proFactory)
                        .processor(processor)
        );

        System.out.println("Start server on port "+ port +"...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 15
public ChunkHeaderBufferedTBaseSerializerFactory(boolean safetyGuaranteed, int outputStreamSize, boolean autoExpand, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.safetyGuaranteed = safetyGuaranteed;
    this.outputStreamSize = outputStreamSize;
    this.autoExpand = autoExpand;
    this.protocolFactory = protocolFactory;
    this.locator = locator;
}
 
Example 16
public CommandHeaderTBaseSerializerFactory(int outputStreamSize) {
    TypeLocator<TBase<?, ?>> commandTbaseLocator = TCommandRegistry.build(Arrays.asList(TCommandType.values()));

    TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
    HeaderTBaseSerializerFactory serializerFactory = new HeaderTBaseSerializerFactory(true, outputStreamSize, protocolFactory, commandTbaseLocator);
    this.factory = new ThreadLocalHeaderTBaseSerializerFactory<HeaderTBaseSerializer>(serializerFactory);
}
 
Example 17
Source Project: drift   Source File: TestDriftNettyMethodInvoker.java    License: Apache License 2.0 5 votes vote down vote up
private static int testProcessor(TProcessor processor, List<ToIntFunction<HostAndPort>> clients)
        throws Exception
{
    try (TServerSocket serverTransport = new TServerSocket(0)) {
        TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
        TTransportFactory transportFactory = new TFramedTransport.Factory();
        TServer server = new TSimpleServer(new Args(serverTransport)
                .protocolFactory(protocolFactory)
                .transportFactory(transportFactory)
                .processor(processor));

        Thread serverThread = new Thread(server::serve);
        try {
            serverThread.start();

            int localPort = serverTransport.getServerSocket().getLocalPort();
            HostAndPort address = HostAndPort.fromParts("localhost", localPort);

            int sum = 0;
            for (ToIntFunction<HostAndPort> client : clients) {
                sum += client.applyAsInt(address);
            }
            return sum;
        }
        finally {
            server.stop();
            serverThread.interrupt();
        }
    }
}
 
Example 18
Source Project: nettythrift   Source File: ProtocolFactorySelector.java    License: Apache License 2.0 5 votes vote down vote up
public TProtocolFactory getProtocolFactory(short head) {
	// SimpleJson的前两个字符为:[" ,而TJSONProtocol的第二个字符为一个数字
	TProtocolFactory fac = protocolFactoryMap.get(head);
	if (logger.isDebugEnabled()) {
		logger.debug("head:{}, getProtocolFactory:{}", head, fac);
	}
	return fac;
}
 
Example 19
Source Project: ikasoa   Source File: ThriftFrameDecoder.java    License: MIT License 5 votes vote down vote up
protected ChannelBuffer tryDecodeUnframedMessage(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer,
		TProtocolFactory inputProtocolFactory) throws TException {

	int messageLength = 0;
	int messageStartReaderIndex = buffer.readerIndex();

	try {
		TNettyTransport decodeAttemptTransport = new TNettyTransport(channel, buffer, TNettyTransportType.UNFRAMED);
		int initialReadBytes = decodeAttemptTransport.getReadByteCount();
		TProtocol inputProtocol = inputProtocolFactory.getProtocol(decodeAttemptTransport);
		inputProtocol.readMessageBegin();
		TProtocolUtil.skip(inputProtocol, TType.STRUCT);
		inputProtocol.readMessageEnd();
		messageLength = decodeAttemptTransport.getReadByteCount() - initialReadBytes;
	} catch (TTransportException | IndexOutOfBoundsException e) {
		return null;
	} finally {
		if (buffer.readerIndex() - messageStartReaderIndex > maxFrameSize)
			Channels.fireExceptionCaught(ctx,
					new TooLongFrameException(String.format("Maximum frame size of %d exceeded .", maxFrameSize)));
		buffer.readerIndex(messageStartReaderIndex);
	}

	if (messageLength <= 0)
		return null;

	ChannelBuffer messageBuffer = extractFrame(buffer, messageStartReaderIndex, messageLength);
	buffer.readerIndex(messageStartReaderIndex + messageLength);
	return messageBuffer;
}
 
Example 20
Source Project: ikasoa   Source File: GeneralFactory.java    License: MIT License 5 votes vote down vote up
/**
 * 获取客户端AsyncService对象
 */
@Override
public AsyncService getAsyncService(TNonblockingTransport transport, String serviceName) throws IkasoaException {
	if (ObjectUtil.isNull(transport))
		throw new IllegalArgumentException("'transport' is null !");
	try {
		return StringUtil.isEmpty(serviceName)
				? new AsyncServiceClientImpl((TProtocolFactory) new TCompactProtocol.Factory(), transport)
				: new AsyncServiceClientImpl(new AsyncMultiplexedProtocolFactory(serviceName), transport);
	} catch (IOException e) {
		throw new IkasoaException(e);
	}
}
 
Example 21
Source Project: pinpoint   Source File: HeaderTBaseSerializerFactory.java    License: Apache License 2.0 5 votes vote down vote up
public HeaderTBaseSerializerFactory(boolean safetyGuaranteed, int outputStreamSize, boolean autoExpand, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.safetyGuaranteed = safetyGuaranteed;
    this.outputStreamSize = outputStreamSize;
    this.autoExpand = autoExpand;
    this.protocolFactory = protocolFactory;
    this.locator = locator;
}
 
Example 22
Source Project: pinpoint   Source File: HttpEchoTestServer.java    License: Apache License 2.0 5 votes vote down vote up
private EchoHttpServerHandler(String path, TProcessor processor, TProtocolFactory protocolFactory) {
    this.path = path;
    this.servlet = new TServlet(processor, protocolFactory) {
        @Override
        protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
            super.service(req, resp);
        }
    };
}
 
Example 23
Source Project: pinpoint   Source File: HeaderTBaseSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new HeaderTBaseSerializer. 
 */
HeaderTBaseSerializer(ResettableByteArrayOutputStream bos, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.baos = bos;
    TIOStreamTransport transport = new TIOStreamTransport(bos);
    this.protocol = protocolFactory.getProtocol(transport);
    this.locator = locator;
}
 
Example 24
Source Project: armeria   Source File: ThriftJacksonSerializers.java    License: Apache License 2.0 5 votes vote down vote up
private static String serializeTBaseLike(Consumer<TProtocol> writer, boolean useNamedEnums) {
    final TMemoryBuffer buffer = new TMemoryBuffer(1024);
    final TProtocolFactory factory = useNamedEnums ? ThriftProtocolFactories.TEXT_NAMED_ENUM
                                                   : ThriftProtocolFactories.TEXT;
    final TProtocol protocol = factory.getProtocol(buffer);
    writer.accept(protocol);
    return new String(buffer.getArray(), 0, buffer.length());
}
 
Example 25
Source Project: pinpoint   Source File: FlinkHeaderTBaseSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new HeaderTBaseSerializer.
 *
 * @param bos
 * @param protocolFactory
 */
public FlinkHeaderTBaseSerializer(ResettableByteArrayOutputStream bos, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.baos = Assert.requireNonNull(bos, "ResettableByteArrayOutputStream");
    this.locator = Assert.requireNonNull(locator, "locator");

    Assert.requireNonNull(protocolFactory, "TProtocolFactory");
    TIOStreamTransport transport = new TIOStreamTransport(bos);
    this.protocol = protocolFactory.getProtocol(transport);


}
 
Example 26
Source Project: hbase   Source File: ThriftServer.java    License: Apache License 2.0 5 votes vote down vote up
protected TServer getTNonBlockingServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString());
  TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
  serverArgs.processor(processor);
  serverArgs.transportFactory(transportFactory);
  serverArgs.protocolFactory(protocolFactory);
  return new TNonblockingServer(serverArgs);
}
 
Example 27
Source Project: hbase   Source File: ThriftServer.java    License: Apache License 2.0 5 votes vote down vote up
protected TProtocolFactory getProtocolFactory() {
  TProtocolFactory protocolFactory;

  if (conf.getBoolean(COMPACT_CONF_KEY, COMPACT_CONF_DEFAULT)) {
    LOG.debug("Using compact protocol");
    protocolFactory = new TCompactProtocol.Factory();
  } else {
    LOG.debug("Using binary protocol");
    protocolFactory = new TBinaryProtocol.Factory();
  }

  return protocolFactory;
}
 
Example 28
Source Project: hbase   Source File: ThriftHttpServlet.java    License: Apache License 2.0 5 votes vote down vote up
public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
    UserGroupInformation serviceUGI, UserGroupInformation httpUGI,
    HBaseServiceHandler handler, boolean securityEnabled, boolean doAsEnabled) {
  super(processor, protocolFactory);
  this.serviceUGI = serviceUGI;
  this.httpUGI = httpUGI;
  this.handler = handler;
  this.securityEnabled = securityEnabled;
  this.doAsEnabled = doAsEnabled;
}
 
Example 29
Source Project: parquet-mr   Source File: TestParquetTBaseScheme.java    License: Apache License 2.0 5 votes vote down vote up
private void createFileForRead() throws Exception {
  final Path fileToCreate = new Path(parquetInputPath+"/names.parquet");

  final Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) fs.delete(fileToCreate, true);

  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, Name.class);

  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  Name n1 = new Name();
  n1.setFirst_name("Alice");
  n1.setLast_name("Practice");
  Name n2 = new Name();
  n2.setFirst_name("Bob");
  n2.setLast_name("Hope");
  Name n3 = new Name();
  n3.setFirst_name("Charlie");
  n3.setLast_name("Horse");

  n1.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n2.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n3.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 
Example 30
Source Project: parquet-mr   Source File: ParquetScroogeSchemeTest.java    License: Apache License 2.0 5 votes vote down vote up
private void writeParquetFile(List<TBase> recordsToWrite, Configuration conf, Path parquetFile) throws IOException, InterruptedException, org.apache.thrift.TException {
  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  Class writeClass = recordsToWrite.get(0).getClass();
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, writeClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  for (TBase recordToWrite : recordsToWrite) {
    recordToWrite.write(protocol);
  }
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}