org.apache.thrift.TSerializer Java Examples

public static final String serializeLoggingEvent(KeyStore keystore, LoggingEvent event) {
  if (null == event) {
    return null;
  }
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  byte[] serialized = null;
  
  try {
    serialized = serializer.serialize(event);
  } catch (TException te) {
    return null;
  }
  
  if (!checkedAESKey) {
    checkedAESKey = true;
    loggingAESKey = keystore.getKey(KeyStore.AES_LOGGING);      
  }
  if (null != loggingAESKey) {
    serialized = CryptoUtils.wrap(loggingAESKey, serialized);
  }
  
  return new String(OrderPreservingBase64.encode(serialized), StandardCharsets.US_ASCII);
}
 

@Deprecated
public static void serialize(TSerializer serializer, TBase struct, DataOutput out)
throws IOException
{
    assert serializer != null;
    assert struct != null;
    assert out != null;
    byte[] bytes;
    try
    {
        bytes = serializer.serialize(struct);
    }
    catch (TException e)
    {
        throw new RuntimeException(e);
    }
    out.writeInt(bytes.length);
    out.write(bytes);
}
 

void pushMetadataMessage(Metadata metadata) throws IOException {
  
  if (null == metadata) {
    pushMetadataMessage(null, null);
    return;
  }
  
  //
  // Compute class/labels Id
  //
  // 128bits
  metadata.setClassId(GTSHelper.classId(this.classKey, metadata.getName()));
  metadata.setLabelsId(GTSHelper.labelsId(this.labelsKey, metadata.getLabels()));
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  try {
    byte[] bytes = new byte[16];
    GTSHelper.fillGTSIds(bytes, 0, metadata.getClassId(), metadata.getLabelsId());
    pushMetadataMessage(bytes, serializer.serialize(metadata));
  } catch (TException te) {
    throw new IOException("Unable to push metadata.");
  }
}
 

@Test
public void testDeserializeColumn() throws Exception {
  Mutation mutation =
      new Mutation(
          MutationType.DELETE,
          TIMESTAMP,
          SOURCE_ID,
          DATA_SOURCE,
          BINLOG_HEADER,
          TABLE,
          getEntity());

  TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
  TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());

  byte[] serialized = serializer.serialize(mutation);

  Mutation deserialized = new Mutation();
  deserializer.deserialize(deserialized, serialized);

  assertEquals(mutation, deserialized);
}
 

public static String pack(GeoXPShape shape) throws WarpScriptException {
  long[] cells = GeoXPLib.getCells(shape);
  
  GTSEncoder encoder = new GTSEncoder();
  
  try {
    for (long cell: cells) {
      encoder.addValue(cell, GeoTimeSerie.NO_LOCATION, GeoTimeSerie.NO_ELEVATION, true);
    }      
  } catch (IOException ioe) {
    throw new WarpScriptException(ioe);
  }
  
  GTSWrapper wrapper = GTSWrapperHelper.fromGTSEncoderToGTSWrapper(encoder, true);
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  try {
    byte[] serialized = serializer.serialize(wrapper);
    
    return new String(OrderPreservingBase64.encode(serialized, 0, serialized.length), StandardCharsets.US_ASCII);
  } catch (TException te) {
    throw new WarpScriptException(te);
  }
}
 

@Test
public void testThrftSerialization() throws TException {
  HMSPathsDumper serDe = genHMSPathsDumper();
  long t1 = System.currentTimeMillis();
  TPathsDump pathsDump = serDe.createPathsDump();
  
  TProtocolFactory protoFactory = useCompact ? new TCompactProtocol.Factory(
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT)
      : new TBinaryProtocol.Factory(true, true,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT);
  byte[] ser = new TSerializer(protoFactory).serialize(pathsDump);
  long serTime = System.currentTimeMillis() - t1;
  System.out.println("Serialization Time: " + serTime + ", " + ser.length);

  t1 = System.currentTimeMillis();
  TPathsDump tPathsDump = new TPathsDump();
  new TDeserializer(protoFactory).deserialize(tPathsDump, ser);
  HMSPaths fromDump = serDe.initializeFromDump(tPathsDump);
  System.out.println("Deserialization Time: " + (System.currentTimeMillis() - t1));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db9.tbl999")), fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db9", "tbl999"}, false));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db9.tbl999")), fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db9", "tbl999", "part99"}, false));
}
 

/**
 * Encodes a thrift object into a binary array.
 *
 * @param tBase Object to encode.
 * @return Encoded object.
 * @throws CodingException If the object could not be encoded.
 */
public static byte[] encodeNonNull(TBase<?, ?> tBase) throws CodingException {
  requireNonNull(tBase);

  try {
    return new TSerializer(PROTOCOL_FACTORY).serialize(tBase);
  } catch (TException e) {
    throw new CodingException("Failed to serialize: " + tBase, e);
  }
}
 

@Override
public String serializeExampleRequest(String serviceName, String methodName,
                                      Object exampleRequest) {
    if (!(exampleRequest instanceof TBase)) {
        return null;
    }

    final TBase<?, ?> exampleTBase = (TBase<?, ?>) exampleRequest;
    final TSerializer serializer = new TSerializer(ThriftProtocolFactories.TEXT);
    try {
        return serializer.toString(exampleTBase, StandardCharsets.UTF_8.name());
    } catch (TException e) {
        throw new Error("should never reach here", e);
    }
}
 

@Override
public @Nullable ByteBuf encodeBody(@NotNull Object body) {
    try {
        TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
        byte[] payloadBytes = serializer.serialize((TBase<?, ?>) body);
        return Unpooled.wrappedBuffer(payloadBytes);
    } catch (TException e) {
        logger.error("Failed to encode {} body", body.getClass().getName(), e);
    }
    return null;
}
 

private Message buildMessage(long timestamp, int timestampTwo, long timestampThree) throws Exception {
    UnitTestMessage thriftMessage = new UnitTestMessage(timestamp, "notimportant", timestampTwo, timestampThree);
    TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
    byte[] data = serializer.serialize(thriftMessage);

    return new Message("test", 0, 0, null, data, timestamp, null);
}
 

@SuppressWarnings("rawtypes")
public static byte[] serialize(TBase baseObject) throws IOException {
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory(maxMessageSize, maxMessageSize));
  try {
    return serializer.serialize(baseObject);
  } catch (TException e) {
    throw new IOException("Error serializing thrift object "
        + baseObject, e);
  }
}
 

public static byte[] toBytes(TBase event) {
	TSerializer tSerializer = tSerializerLocal.get();
	byte[] data;
	try {
		data = tSerializer.serialize(event);
	} catch (TException e) {
		throw new RuntimeException("fail to serialize thrift object of type " + event.getClass());
	}
	return data;
}
 

public ThriftLogGenerator(
    String logDir,
    String logFileName,
    int bytesPerSecond,
    int maxNumOfMessages,
    int numOfBytesPerLogFile,
    int numOfLogFiles,
    int payloadSizeInBytes) throws Exception {
  UUID uuid = UUID.randomUUID();
  this.logGeneratorId = ByteBuffer.wrap(new byte[16])
      .putLong(uuid.getMostSignificantBits())
      .putLong(uuid.getLeastSignificantBits())
      .array();
  this.logDir = logDir;
  this.logFileName = logFileName;

  this.rateLimiter = RateLimiter.create(bytesPerSecond);
  this.logger = new SimpleThriftLogger<>(FilenameUtils.concat(logDir, logFileName));
  this.random = new Random();

  this.maxNumOfMessages = maxNumOfMessages;
  this.numOfBytesPerLogFile = numOfBytesPerLogFile;
  this.numOfLogFiles = numOfLogFiles;
  this.payloadSizeInBytes = payloadSizeInBytes;

  this.numOfLogMessageSequence = 100;
  this.nextSequenceNums = Lists.newArrayListWithExpectedSize(numOfLogMessageSequence);
  this.sequenceIds = Lists.newArrayListWithExpectedSize(numOfLogMessageSequence);

  this.serializer = new TSerializer(new TBinaryProtocol.Factory());
}
 

private AbstractHandler mockCoordinatorResponse(
    IScheduledTask task,
    String pollResponse,
    Map<String, String> params) {

  return new AbstractHandler() {
    @Override
    public void handle(
        String target,
        Request baseRequest,
        HttpServletRequest request,
        HttpServletResponse response) throws IOException {
      try {
        String taskKey = slaManager.getTaskKey(task);
        String query = Joiner
            .on("=")
            .join(SlaManager.TASK_PARAM, URLEncoder.encode(taskKey, "UTF-8"));

        String taskConfig = new TSerializer(new TSimpleJSONProtocol.Factory())
            .toString(task.newBuilder());
        JsonObject jsonBody = new JsonObject();
        jsonBody.add("taskConfig", new JsonParser().parse(taskConfig));
        jsonBody.addProperty(SlaManager.TASK_PARAM, taskKey);
        params.forEach(jsonBody::addProperty);
        String body = new Gson().toJson(jsonBody);

        if (request.getQueryString().equals(query)
            && request.getReader().lines().collect(Collectors.joining()).equals(body)) {
          createResponse(baseRequest, response, pollResponse);
        }
        coordinatorResponded.countDown();
      } catch (TException e) {
        fail();
      }
    }
  };
}
 

private static String serialize(Op op) {
  try {
    String unformattedJson =
        new String(new TSerializer(new TJSONProtocol.Factory()).serialize(op), UTF_8);

    // Pretty print the json for easier review of diffs.
    return new GsonBuilder().setPrettyPrinting().create()
        .toJson(new JsonParser().parse(unformattedJson)) + "\n";
  } catch (TException e) {
    throw new RuntimeException(e);
  }
}
 

public static byte[] serialize(ThriftProtocol protocol, TBase<?, ?> source)
    throws ThriftException {
  TSerializer serializer = new TSerializer(getProtocolFactory(protocol));
  try {
    return serializer.serialize(source);
  } catch (TException e) {
    throw new ThriftException(e);
  }
}
 

public static ByteBuffer serializeToByteBuffer(ThriftProtocol protocol, TBase<?, ?> source)
    throws ThriftException {
  TSerializer serializer = new TSerializer(getProtocolFactory(protocol));
  try {
    return ByteBuffer.wrap(serializer.serialize(source));
  } catch (TException e) {
    throw new ThriftException(e);
  }
}
 

public static String thriftToDebugJson(TBase<?, ?> thriftObject) {
  TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());
  try {
    return new String(serializer.serialize(thriftObject));
  } catch (TException e) {
    LOGGER.error(
        e,
        String.format(
            "Failed trying to serialize type [%s] to debug JSON.",
            thriftObject.getClass().getName()));
    return "FAILED_TO_DESERIALIZE";
  }
}
 

public static byte[] thriftSerialize(TBase t) {
    try {
        TSerializer ser = getSer();
        return ser.serialize(t);
    } catch (TException e) {
        LOG.error("Failed to serialize to thrift: ", e);
        throw new RuntimeException(e);
    }
}
 

@Override
public byte[] serialize(Object object) {
    try {
        return Utils.gzip(new TSerializer().serialize((TBase) object));
    } catch (TException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public byte[] serialize(Object object) {
    try {
        return new TSerializer().serialize((TBase) object);
    } catch (TException e) {
        throw new RuntimeException(e);
    }
}
 

public static synchronized boolean offer(WebCallRequest request) {    
  try {
    //
    // Initialize service if not done yet
    //
    
    if (!initialized) {
      initialize();
    }
  
    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    
    byte[] value = serializer.serialize(request);
    
    //
    // Wrap data if AES key is defined
    //
    
    if (null != aesKey) {
      value = CryptoUtils.wrap(aesKey, value);               
    }
    
    //
    // Compute MAC if the SipHash key is defined
    //
    
    if (null != siphashKey) {
      value = CryptoUtils.addMAC(siphashKey, value);
    }

    KeyedMessage<byte[], byte[]> message = new KeyedMessage<byte[], byte[]>(topic, value);
    
    producer.send(message);
    
    return true;
  } catch (Exception e) {
    return false;
  }
}
 

public byte[] encode(Transaction tx) throws IOException {
  TTransaction thriftTx = TransactionConverterUtils.wrap(tx);
  TSerializer serializer = new TSerializer();
  try {
    return serializer.serialize(thriftTx);
  } catch (TException te) {
    throw new IOException(te);
  }
}
 

public static <T extends TBase> String ThriftToJSON(T thrift) {
  TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());
  try {
    return serializer.toString(thrift);
  } catch (TException e) {
  }
  throw new IllegalStateException("Convert to json failed : " + thrift);
}
 

private byte[] encodeOctoHeader(Object message) throws TException {
    TSerializer serializer = new TSerializer();
    Header header;
    if (message instanceof DefaultRequest) {
        DefaultRequest request = (DefaultRequest) message;
        header = MetaUtil.convertRequestToHeader(request);
    } else if (message instanceof DefaultResponse) {
        DefaultResponse response = (DefaultResponse) message;
        header = MetaUtil.convertResponseToHeader(response);
    } else {
        throw new ProtocolException("Serialize header, but object is invalid.");
    }
    return serializer.serialize(header);
}
 

@Override
protected TSerializer initialValue() {
  return new TSerializer(new TCompactProtocol.Factory());
}
 

@Override
protected synchronized TSerializer initialValue() {
	return new TSerializer();
}
 

public static final String secure(String key, String script) throws WarpScriptException {
  SecureScript sscript = new SecureScript();
  sscript.setTimestamp(System.currentTimeMillis());
  sscript.setKey(key);

  byte[] scriptBytes = script.getBytes(StandardCharsets.UTF_8);
  
  // Check if we should compress the script or not
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  
  boolean compress = false;
  
  try {
    GZIPOutputStream gzos = new GZIPOutputStream(baos);
    gzos.write(scriptBytes);
    gzos.close();
    byte[] gzipped = baos.toByteArray();
    if (gzipped.length < scriptBytes.length) {
      compress = true;
      scriptBytes = gzipped;
    }
  } catch (IOException ioe) {              
  }
  
  sscript.setCompressed(compress);
  sscript.setScript(scriptBytes);
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  try {
    byte[] serialized = serializer.serialize(sscript);
    // TODO(hbs): encrypt script
    
    synchronized(SECURE.class) {
      if (null == aesKey) {
        try {
          aesKey = WarpDist.getKeyStore().getKey(KeyStore.AES_SECURESCRIPTS);
        } catch (Throwable t) {
          // Catch NoClassDefFoundError
        }
      }
    }
    
    if (null == aesKey) {
      throw new WarpScriptException("Missing secure script encryption key.");
    }
    
    byte[] wrapped = CryptoUtils.wrap(aesKey, serialized);
    
    String encoded = new String(OrderPreservingBase64.encode(wrapped), StandardCharsets.US_ASCII);
    
    return encoded;
  } catch (TException te) {
    throw new WarpScriptException("Unable to secure script.", te);
  }

}
 

private boolean coordinatorAllows(
    IScheduledTask task,
    String taskKey,
    ICoordinatorSlaPolicy slaPolicy,
    Map<String, String> params)
    throws InterruptedException, ExecutionException, TException {

  LOG.info("Checking coordinator: {} for task: {}", slaPolicy.getCoordinatorUrl(), taskKey);

  String taskConfig = new TSerializer(new TSimpleJSONProtocol.Factory())
      .toString(task.newBuilder());
  JsonObject jsonBody = new JsonObject();
  jsonBody.add("taskConfig", new JsonParser().parse(taskConfig));
  jsonBody.addProperty(TASK_PARAM, taskKey);
  params.forEach(jsonBody::addProperty);

  Response response = httpClient.preparePost(slaPolicy.getCoordinatorUrl())
      .setQueryParams(ImmutableList.of(new Param(TASK_PARAM, taskKey)))
      .setBody(new Gson().toJson(jsonBody))
      .execute()
      .get();

  if (response.getStatusCode() != HttpConstants.ResponseStatusCodes.OK_200) {
    LOG.error("Request failed to coordinator: {} for task: {}. Response: {}",
        slaPolicy.getCoordinatorUrl(),
        taskKey,
        response.getStatusCode());
    incrementErrorCount(USER_ERRORS_STAT_NAME, taskKey);
    return false;
  }

  successCounter.incrementAndGet();
  String json = response.getResponseBody();
  LOG.info("Got response: {} from {} for task: {}",
      json,
      slaPolicy.getCoordinatorUrl(),
      taskKey);

  Map<String, Boolean> result = new Gson().fromJson(
      json,
      new TypeReference<Map<String, Boolean>>() { }.getType());

  return result.get(slaPolicy.isSetStatusKey() ? slaPolicy.getStatusKey() : "drain");
}
 

public byte[] toBytes() throws IOException {
  HyperLogLogPlusParameters params = new HyperLogLogPlusParameters();
  
  params.setInitTime(this.initTime);
  params.setP((byte) this.p);
  params.setPprime((byte) this.pprime);
  params.setSparse(Format.SPARSE == this.format);
  
  if (null != this.key) {
    params.setKey(this.key);
  }
  
  if (Format.SPARSE == this.format) {      
    // Trigger a merge
    merge();
    // Output the sparse list size
    params.setSparseListLen(sparse_list_len);
    params.setSparseList(sparse_list);
  } else {
    // Output the registers
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    GZIPOutputStream gzos = new GZIPOutputStream(baos);
    gzos.write(this.M);
    gzos.close();
    
    byte[] gzipped = baos.toByteArray();
    
    if (gzipped.length < this.M.length) {
      params.setRegisters(gzipped);
      params.setGzipped(true);
    } else {
      params.setRegisters(this.M);
      params.setGzipped(false);
    }
  }
  
  try {
    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    byte[] ser = serializer.serialize(params);
    return ser;
  } catch (TException te) {
    throw new IOException(te);
  }
}