org.apache.thrift.protocol.TProtocolFactory Java Examples

/**
 * Returns the {@link TProtocolFactory} for the specified {@link SerializationFormat}.
 *
 * @throws IllegalArgumentException if the specified {@link SerializationFormat} is not for Thrift
 */
public static TProtocolFactory get(SerializationFormat serializationFormat) {
    requireNonNull(serializationFormat, "serializationFormat");

    if (serializationFormat == ThriftSerializationFormats.BINARY) {
        return BINARY;
    }

    if (serializationFormat == ThriftSerializationFormats.COMPACT) {
        return COMPACT;
    }

    if (serializationFormat == ThriftSerializationFormats.JSON) {
        return JSON;
    }

    if (serializationFormat == ThriftSerializationFormats.TEXT) {
        return TEXT;
    }

    if (serializationFormat == ThriftSerializationFormats.TEXT_NAMED_ENUM) {
        return TEXT_NAMED_ENUM;
    }

    throw new IllegalArgumentException("non-Thrift serializationFormat: " + serializationFormat);
}
 

@Override
protected void messageReceived(ChannelHandlerContext ctx, ThriftMessage message) throws Exception {
	ByteBuf buffer = message.getContent();
	logger.debug("msg.content:: {}", buffer);
	try {
		TNettyTransport transport = new TNettyTransport(ctx.channel(), buffer);
		TProtocolFactory protocolFactory = message.getProtocolFactory();
		TProtocol protocol = protocolFactory.getProtocol(transport);
		serverDef.nettyProcessor.process(ctx, protocol, protocol,
				new DefaultWriterListener(message, transport, ctx, serverDef));
	} catch (Throwable ex) {
		int refCount = buffer.refCnt();
		if (refCount > 0) {
			buffer.release(refCount);
		}
		throw ex;
	}
}
 

public static TProtocolFactory getProtocolFactory(ThriftProtocol protocol) {
  // TODO(ruibm): Check whether the Factories are thread safe so we can static initialize
  // them just once.
  switch (protocol) {
    case JSON:
      return new TJSONProtocol.Factory();

    case COMPACT:
      return new TCompactProtocol.Factory();

    case BINARY:
      return new TBinaryProtocol.Factory();

    default:
      throw new IllegalArgumentException(
          String.format("Unknown ThriftProtocol [%s].", protocol.toString()));
  }
}
 

protected TServer getTThreadPoolServer(TProtocolFactory protocolFactory, TProcessor processor,
    TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws Exception {
  LOG.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
  // Thrift's implementation uses '0' as a placeholder for 'use the default.'
  int backlog = conf.getInt(BACKLOG_CONF_KEY, BACKLOG_CONF_DEAFULT);
  int readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
      THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
  TServerTransport serverTransport = new TServerSocket(
      new TServerSocket.ServerSocketTransportArgs().
          bindAddr(inetSocketAddress).backlog(backlog).
          clientTimeout(readTimeout));

  TBoundedThreadPoolServer.Args serverArgs =
      new TBoundedThreadPoolServer.Args(serverTransport, conf);
  serverArgs.processor(processor).transportFactory(transportFactory)
      .protocolFactory(protocolFactory);
  return new TBoundedThreadPoolServer(serverArgs, metrics);
}
 

protected TServer getTThreadedSelectorServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase ThreadedSelector Thrift server on " + inetSocketAddress.toString());
  TThreadedSelectorServer.Args serverArgs =
      new HThreadedSelectorServerArgs(serverTransport, conf);
  int queueSize = conf.getInt(TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY,
      TBoundedThreadPoolServer.DEFAULT_MAX_QUEUED_REQUESTS);
  CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(queueSize), metrics);
  int workerThreads = conf.getInt(TBoundedThreadPoolServer.MAX_WORKER_THREADS_CONF_KEY,
      serverArgs.getWorkerThreads());
  int selectorThreads = conf.getInt(THRIFT_SELECTOR_NUM, serverArgs.getSelectorThreads());
  serverArgs.selectorThreads(selectorThreads);
  ExecutorService executorService = createExecutor(
      callQueue, workerThreads, workerThreads);
  serverArgs.executorService(executorService).processor(processor)
      .transportFactory(transportFactory).protocolFactory(protocolFactory);
  return new TThreadedSelectorServer(serverArgs);
}
 

protected TServer getTHsHaServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
  THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
  int queueSize = conf.getInt(TBoundedThreadPoolServer.MAX_QUEUED_REQUESTS_CONF_KEY,
      TBoundedThreadPoolServer.DEFAULT_MAX_QUEUED_REQUESTS);
  CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(queueSize), metrics);
  int workerThread = conf.getInt(TBoundedThreadPoolServer.MAX_WORKER_THREADS_CONF_KEY,
      serverArgs.getMaxWorkerThreads());
  ExecutorService executorService = createExecutor(
      callQueue, workerThread, workerThread);
  serverArgs.executorService(executorService).processor(processor)
      .transportFactory(transportFactory).protocolFactory(protocolFactory);
  return new THsHaServer(serverArgs);
}
 

private static TServer startNewThreadPoolServer(final TServerTransport transport) {
  final TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();

  final CustomHandler customHandler = new CustomHandler();
  final CustomService.Processor<CustomService.Iface> customProcessor = new CustomService.Processor<CustomService.Iface>(customHandler);

  final TThreadPoolServer.Args args = new TThreadPoolServer
    .Args(transport)
    .processorFactory(new TProcessorFactory(customProcessor))
    .protocolFactory(protocolFactory)
    .minWorkerThreads(5)
    .maxWorkerThreads(10);

  final TServer server = new TThreadPoolServer(args);
  new Thread(new Runnable() {
    @Override
    public void run() {
      server.serve();
    }
  }).start();

  return server;
}
 

private void startNewThreadPoolServer() throws Exception {
  final TServerTransport transport = new TServerSocket(port);
  final TProtocolFactory protocolFactory = new TBinaryProtocol.Factory();
  // TTransportFactory transportFactory = new TFramedTransport.Factory();

  final CustomHandler customHandler = new CustomHandler();
  final TProcessor customProcessor = new CustomService.Processor<CustomService.Iface>(customHandler);

  final TThreadPoolServer.Args args = new TThreadPoolServer.Args(transport)
    .processorFactory(new TProcessorFactory(customProcessor))
    .protocolFactory(protocolFactory)
    // .transportFactory(transportFactory)
    .minWorkerThreads(5)
    // .executorService(Executors.newCachedThreadPool())
    .maxWorkerThreads(10);

  server = new TThreadPoolServer(args);
  new Thread(new Runnable() {
    @Override
    public void run() {
      server.serve();
    }
  }).start();
}
 

/**
 * 阻塞式、多线程处理
 *
 * @param args
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) {
    try {
        int port = Integer.valueOf(prop.get("dkml.port"));
        //设置传输通道，普通通道
        TServerTransport serverTransport = new TServerSocket(port);
        //使用高密度二进制协议
        TProtocolFactory proFactory = new TCompactProtocol.Factory();
        //设置处理器
        TProcessor processor = new DKML.Processor(new DKMLImpl());
        //创建服务器
        TServer server = new TThreadPoolServer(
                new TThreadPoolServer.Args(serverTransport)
                        .protocolFactory(proFactory)
                        .processor(processor)
        );

        System.out.println("Start server on port "+ port +"...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

private <T extends TBase<?,?>> Path createFile(T... tObjs) throws IOException, InterruptedException, TException  {
  final Path fileToCreate = new Path("target/test/TestThriftToParquetFileWriter/"+tObjs[0].getClass()+".parquet");
  LOG.info("File created: {}", fileToCreate.toString());
  Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) {
    fs.delete(fileToCreate, true);

  }
  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, (Class<? extends TBase<?, ?>>) tObjs[0].getClass());

  for(T tObj:tObjs) {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

    tObj.write(protocol);

    w.write(new BytesWritable(baos.toByteArray()));
  }
  w.close();

  return fileToCreate;
}
 

/**
 * Returns the {@link SerializationFormat} for the specified {@link TProtocolFactory}.
 *
 * @throws IllegalArgumentException if the specified {@link TProtocolFactory} is not known by this class
 */
public static SerializationFormat toSerializationFormat(TProtocolFactory protoFactory) {
    requireNonNull(protoFactory, "protoFactory");

    if (protoFactory instanceof TBinaryProtocol.Factory) {
        return ThriftSerializationFormats.BINARY;
    } else if (protoFactory instanceof TCompactProtocol.Factory) {
        return ThriftSerializationFormats.COMPACT;
    } else if (protoFactory instanceof TJSONProtocol.Factory) {
        return ThriftSerializationFormats.JSON;
    } else if (protoFactory instanceof TTextProtocolFactory) {
        final TTextProtocolFactory factory = (TTextProtocolFactory) protoFactory;
        return factory.usesNamedEnums() ? ThriftSerializationFormats.TEXT_NAMED_ENUM
                                        : ThriftSerializationFormats.TEXT;
    } else {
        throw new IllegalArgumentException(
                "unsupported TProtocolFactory: " + protoFactory.getClass().getName());
    }
}
 

public ThriftMessageParser(SecorConfig config)
        throws InstantiationException, IllegalAccessException,
        ClassNotFoundException {
    super(config);
    TProtocolFactory protocolFactory = null;
    String protocolName = mConfig.getThriftProtocolClass();
    
    if (StringUtils.isNotEmpty(protocolName)) {
        String factoryClassName = protocolName.concat("$Factory");
        protocolFactory = ((Class<? extends TProtocolFactory>) Class.forName(factoryClassName)).newInstance();
    } else
        protocolFactory = new TBinaryProtocol.Factory();
    
    mDeserializer = new TDeserializer(protocolFactory);
    mThriftPath = new ThriftPath(mConfig.getMessageTimestampName(),(short) mConfig.getMessageTimestampId());
    mTimestampType = mConfig.getMessageTimestampType();
}
 

protected TProtocolFactory getProtocolFactory() {
  TProtocolFactory protocolFactory;

  if (conf.getBoolean(COMPACT_CONF_KEY, COMPACT_CONF_DEFAULT)) {
    LOG.debug("Using compact protocol");
    protocolFactory = new TCompactProtocol.Factory();
  } else {
    LOG.debug("Using binary protocol");
    protocolFactory = new TBinaryProtocol.Factory();
  }

  return protocolFactory;
}
 

public TProtocolFactory getProtocolFactory(short head) {
	// SimpleJson的前两个字符为：[" ，而TJSONProtocol的第二个字符为一个数字
	TProtocolFactory fac = protocolFactoryMap.get(head);
	if (logger.isDebugEnabled()) {
		logger.debug("head:{}, getProtocolFactory:{}", head, fac);
	}
	return fac;
}
 

private void createFileForRead() throws Exception {
  final Path fileToCreate = new Path(parquetInputPath + "/names.parquet");

  final Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) fs.delete(fileToCreate, true);

  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, Name.class);

  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  Name n1 = new Name();
  n1.setFirst_name("Alice");
  n1.setLast_name("Practice");
  Name n2 = new Name();
  n2.setFirst_name("Bob");
  n2.setLast_name("Hope");
  Name n3 = new Name();
  n3.setFirst_name("Charlie");
  n3.setLast_name("Horse");

  n1.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n2.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n3.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 

public ServerConfig(int port, TProtocolFactory tProtocolFactory,
                    int minWorkerThread, int maxWorkerThread) {
  this.port = port;
  this.tProtocolFactory = tProtocolFactory;
  this.minWorkerThread = minWorkerThread;
  this.maxWorkerThread = maxWorkerThread;
}
 

public static Class<TProtocolFactory> getTProtocolFactoryClass(Configuration conf) {
  final String tProtocolClassName = conf.get(PARQUET_PROTOCOL_CLASS);
  if (tProtocolClassName == null) {
    throw new BadConfigurationException("the protocol class conf is missing in job conf at " + PARQUET_PROTOCOL_CLASS);
  }
  try {
    @SuppressWarnings("unchecked")
    Class<TProtocolFactory> tProtocolFactoryClass = (Class<TProtocolFactory>)Class.forName(tProtocolClassName + "$Factory");
    return tProtocolFactoryClass;
  } catch (ClassNotFoundException e) {
    throw new BadConfigurationException("the Factory for class " + tProtocolClassName + " in job conf at " + PARQUET_PROTOCOL_CLASS + " could not be found", e);
  }
}
 

/**
 * 阻塞式、多线程处理
 *
 * @param args
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) {
    try {
        int port = Integer.valueOf(prop.get("dkgraphx.port"));
        //设置传输通道，普通通道
        TServerTransport serverTransport = new TServerSocket(port);

        //使用高密度二进制协议
        TProtocolFactory proFactory = new TCompactProtocol.Factory();

        //设置处理器
        TProcessor processor = new DKGraphx.Processor(new DKGraphxImpl());

        //创建服务器
        TServer server = new TThreadPoolServer(
                new TThreadPoolServer.Args(serverTransport)
                        .protocolFactory(proFactory)
                        .processor(processor)
        );

        System.out.println("Start server on port "+ port +"...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

public ChunkHeaderBufferedTBaseSerializerFactory(boolean safetyGuaranteed, int outputStreamSize, boolean autoExpand, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.safetyGuaranteed = safetyGuaranteed;
    this.outputStreamSize = outputStreamSize;
    this.autoExpand = autoExpand;
    this.protocolFactory = protocolFactory;
    this.locator = locator;
}
 

public ThriftBytesWriteSupport(TProtocolFactory protocolFactory, Class<? extends TBase<?, ?>> thriftClass, boolean buffered, FieldIgnoredHandler errorHandler) {
  super();
  this.protocolFactory = protocolFactory;
  this.thriftClass = thriftClass;
  this.buffered = buffered;
  this.errorHandler = errorHandler;
  if (!buffered && errorHandler != null) {
    throw new IllegalArgumentException("Only buffered protocol can use error handler for now");
  }
}
 

private void writeParquetFile(List<TBase> recordsToWrite, Configuration conf, Path parquetFile) throws IOException, InterruptedException, org.apache.thrift.TException {
  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  Class writeClass = recordsToWrite.get(0).getClass();
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, writeClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  for (TBase recordToWrite : recordsToWrite) {
    recordToWrite.write(protocol);
  }
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 

private EchoHttpServerHandler(String path, TProcessor processor, TProtocolFactory protocolFactory) {
    this.path = path;
    this.servlet = new TServlet(processor, protocolFactory) {
        @Override
        protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
            super.service(req, resp);
        }
    };
}
 

public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
    UserGroupInformation serviceUGI, UserGroupInformation httpUGI,
    HBaseServiceHandler handler, boolean securityEnabled, boolean doAsEnabled) {
  super(processor, protocolFactory);
  this.serviceUGI = serviceUGI;
  this.httpUGI = httpUGI;
  this.handler = handler;
  this.securityEnabled = securityEnabled;
  this.doAsEnabled = doAsEnabled;
}
 

/**
 * 获取客户端AsyncService对象
 */
@Override
public AsyncService getAsyncService(TNonblockingTransport transport, String serviceName) throws IkasoaException {
	if (ObjectUtil.isNull(transport))
		throw new IllegalArgumentException("'transport' is null !");
	try {
		return StringUtil.isEmpty(serviceName)
				? new AsyncServiceClientImpl((TProtocolFactory) new TCompactProtocol.Factory(), transport)
				: new AsyncServiceClientImpl(new AsyncMultiplexedProtocolFactory(serviceName), transport);
	} catch (IOException e) {
		throw new IkasoaException(e);
	}
}
 

protected ChannelBuffer tryDecodeUnframedMessage(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer,
		TProtocolFactory inputProtocolFactory) throws TException {

	int messageLength = 0;
	int messageStartReaderIndex = buffer.readerIndex();

	try {
		TNettyTransport decodeAttemptTransport = new TNettyTransport(channel, buffer, TNettyTransportType.UNFRAMED);
		int initialReadBytes = decodeAttemptTransport.getReadByteCount();
		TProtocol inputProtocol = inputProtocolFactory.getProtocol(decodeAttemptTransport);
		inputProtocol.readMessageBegin();
		TProtocolUtil.skip(inputProtocol, TType.STRUCT);
		inputProtocol.readMessageEnd();
		messageLength = decodeAttemptTransport.getReadByteCount() - initialReadBytes;
	} catch (TTransportException | IndexOutOfBoundsException e) {
		return null;
	} finally {
		if (buffer.readerIndex() - messageStartReaderIndex > maxFrameSize)
			Channels.fireExceptionCaught(ctx,
					new TooLongFrameException(String.format("Maximum frame size of %d exceeded .", maxFrameSize)));
		buffer.readerIndex(messageStartReaderIndex);
	}

	if (messageLength <= 0)
		return null;

	ChannelBuffer messageBuffer = extractFrame(buffer, messageStartReaderIndex, messageLength);
	buffer.readerIndex(messageStartReaderIndex + messageLength);
	return messageBuffer;
}
 

public HeaderTBaseSerializerFactory(boolean safetyGuaranteed, int outputStreamSize, boolean autoExpand, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.safetyGuaranteed = safetyGuaranteed;
    this.outputStreamSize = outputStreamSize;
    this.autoExpand = autoExpand;
    this.protocolFactory = protocolFactory;
    this.locator = locator;
}
 

protected TServer getTNonBlockingServer(TNonblockingServerTransport serverTransport,
    TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory,
    InetSocketAddress inetSocketAddress) {
  LOG.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString());
  TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
  serverArgs.processor(processor);
  serverArgs.transportFactory(transportFactory);
  serverArgs.protocolFactory(protocolFactory);
  return new TNonblockingServer(serverArgs);
}
 

private static String serializeTBaseLike(Consumer<TProtocol> writer, boolean useNamedEnums) {
    final TMemoryBuffer buffer = new TMemoryBuffer(1024);
    final TProtocolFactory factory = useNamedEnums ? ThriftProtocolFactories.TEXT_NAMED_ENUM
                                                   : ThriftProtocolFactories.TEXT;
    final TProtocol protocol = factory.getProtocol(buffer);
    writer.accept(protocol);
    return new String(buffer.getArray(), 0, buffer.length());
}
 

/**
 * Create a new HeaderTBaseSerializer. 
 */
HeaderTBaseSerializer(ResettableByteArrayOutputStream bos, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.baos = bos;
    TIOStreamTransport transport = new TIOStreamTransport(bos);
    this.protocol = protocolFactory.getProtocol(transport);
    this.locator = locator;
}
 

/**
 * Create a new HeaderTBaseSerializer.
 *
 * @param bos
 * @param protocolFactory
 */
public FlinkHeaderTBaseSerializer(ResettableByteArrayOutputStream bos, TProtocolFactory protocolFactory, TypeLocator<TBase<?, ?>> locator) {
    this.baos = Assert.requireNonNull(bos, "ResettableByteArrayOutputStream");
    this.locator = Assert.requireNonNull(locator, "locator");

    Assert.requireNonNull(protocolFactory, "TProtocolFactory");
    TIOStreamTransport transport = new TIOStreamTransport(bos);
    this.protocol = protocolFactory.getProtocol(transport);


}