org.apache.thrift.TDeserializer Java Examples

@Deprecated
public static void deserialize(TDeserializer deserializer, TBase struct, DataInput in)
throws IOException
{
    assert deserializer != null;
    assert struct != null;
    assert in != null;
    byte[] bytes = new byte[in.readInt()];
    in.readFully(bytes);
    try
    {
        deserializer.deserialize(struct, bytes);
    }
    catch (TException ex)
    {
        throw new IOException(ex);
    }
}
 

public static final LoggingEvent unwrapLog(byte[] key, String logmsg) {    
  try {
    byte[] data = OrderPreservingBase64.decode(logmsg.getBytes(StandardCharsets.US_ASCII));
    
    data = CryptoUtils.unwrap(key, data);
    
    if (null == data) {
      return null;
    }
    
    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
    LoggingEvent event = new LoggingEvent();
    try {
      deserializer.deserialize(event, data);
    } catch (TException te) {
      return null;
    }
    
    return event;      
  } catch (Exception e) {
    return null;
  }
}
 

@Test
public void testDeserializeColumn() throws Exception {
  Mutation mutation =
      new Mutation(
          MutationType.DELETE,
          TIMESTAMP,
          SOURCE_ID,
          DATA_SOURCE,
          BINLOG_HEADER,
          TABLE,
          getEntity());

  TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
  TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());

  byte[] serialized = serializer.serialize(mutation);

  Mutation deserialized = new Mutation();
  deserializer.deserialize(deserialized, serialized);

  assertEquals(mutation, deserialized);
}
 

/**
 * Reads in a list of rule keys from a file
 *
 * @param logPath The path to the file
 * @return A list of FullRuleKey objects from the file
 * @throws IOException Could not read the file
 * @throws TException Could not deserialize an entry from the file
 */
public static List<FullRuleKey> readRuleKeys(Path logPath) throws IOException, TException {
  ByteBuffer lengthBuf = ByteBuffer.allocate(4);
  List<FullRuleKey> ret = new ArrayList<>();
  try (FileInputStream logFileStream = new FileInputStream(logPath.toAbsolutePath().toString())) {
    while (logFileStream.available() > 0) {
      logFileStream.read(lengthBuf.array());
      int length = lengthBuf.getInt();
      lengthBuf.rewind();

      byte[] serialized = new byte[length];
      logFileStream.read(serialized);
      TDeserializer serializer = new TDeserializer(new TCompactProtocol.Factory());
      FullRuleKey ruleKey = new FullRuleKey();
      serializer.deserialize(ruleKey, serialized);
      ret.add(ruleKey);
    }
    return ret;
  }
}
 

public ThriftMessageParser(SecorConfig config)
        throws InstantiationException, IllegalAccessException,
        ClassNotFoundException {
    super(config);
    TProtocolFactory protocolFactory = null;
    String protocolName = mConfig.getThriftProtocolClass();
    
    if (StringUtils.isNotEmpty(protocolName)) {
        String factoryClassName = protocolName.concat("$Factory");
        protocolFactory = ((Class<? extends TProtocolFactory>) Class.forName(factoryClassName)).newInstance();
    } else
        protocolFactory = new TBinaryProtocol.Factory();
    
    mDeserializer = new TDeserializer(protocolFactory);
    mThriftPath = new ThriftPath(mConfig.getMessageTimestampName(),(short) mConfig.getMessageTimestampId());
    mTimestampType = mConfig.getMessageTimestampType();
}
 

private static StringBuilder bytesToString(StringBuilder sb, byte[] bytes) {
  TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
  GTSWrapper wrapper = new GTSWrapper();

  try {
    deser.deserialize(wrapper, bytes);

    return wrapperToString(sb, wrapper);
  } catch (TException te) {
    if (null == sb) {
      sb = new StringBuilder();
    }
    sb.append("b64:");
    sb.append(Base64.encodeBase64URLSafeString(bytes));
    return sb;
  }    
}
 

/**
 * try to decode an encoder from its opb64 string representation or its byte array representation.
 *
 * @param o string, encoder, or byte array
 * @return a GTSDecoder object
 * @throws WarpScriptException
 */
private GTSDecoder getDecoderFromObject(Object o) throws WarpScriptException {
  GTSDecoder decoder;
  if (o instanceof GTSEncoder) {
    decoder = ((GTSEncoder) o).getUnsafeDecoder(false);
  } else {
    try {
      byte[] bytes = o instanceof String ? OrderPreservingBase64.decode(o.toString().getBytes(StandardCharsets.US_ASCII)) : (byte[]) o;
      TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
      GTSWrapper wrapper = new GTSWrapper();
      deser.deserialize(wrapper, bytes);
      decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);
    } catch (TException te) {
      throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
    }
  }
  return decoder;
}
 

@Test
public void testThrftSerialization() throws TException {
  HMSPathsDumper serDe = genHMSPathsDumper();
  long t1 = System.currentTimeMillis();
  TPathsDump pathsDump = serDe.createPathsDump();
  
  TProtocolFactory protoFactory = useCompact ? new TCompactProtocol.Factory(
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT)
      : new TBinaryProtocol.Factory(true, true,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT,
      ServiceConstants.ClientConfig.SENTRY_HDFS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT);
  byte[] ser = new TSerializer(protoFactory).serialize(pathsDump);
  long serTime = System.currentTimeMillis() - t1;
  System.out.println("Serialization Time: " + serTime + ", " + ser.length);

  t1 = System.currentTimeMillis();
  TPathsDump tPathsDump = new TPathsDump();
  new TDeserializer(protoFactory).deserialize(tPathsDump, ser);
  HMSPaths fromDump = serDe.initializeFromDump(tPathsDump);
  System.out.println("Deserialization Time: " + (System.currentTimeMillis() - t1));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db9.tbl999")), fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db9", "tbl999"}, false));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db9.tbl999")), fromDump.findAuthzObject(new String[]{"user", "hive", "warehouse", "db9", "tbl999", "part99"}, false));
}
 

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
  Object top = stack.pop();
  
  if (!(top instanceof String) && !(top instanceof byte[])) {
    throw new WarpScriptException(getName() + " operates on a string or byte array.");
  }
  
  byte[] bytes = top instanceof String ? OrderPreservingBase64.decode(top.toString().getBytes(StandardCharsets.US_ASCII)) : (byte[]) top;
  
  TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
  
  try {
    GTSWrapper wrapper = new GTSWrapper();      
    deser.deserialize(wrapper, bytes);

    stack.push(GTSWrapperHelper.fromGTSWrapperToGTSEncoder(wrapper));      
  } catch (TException te) {
    throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
  } catch (IOException ioe) {
    throw new WarpScriptException(getName() + " failed to unwrap encoder.", ioe);
  }      

  return stack;
}
 

private static Op deserialize(String serializedOp) {
  try {
    Op op = new Op();

    String nonPrettyJson = new GsonBuilder().create()
        .toJson(new JsonParser().parse(serializedOp));

    new TDeserializer(new TJSONProtocol.Factory())
        .deserialize(op, nonPrettyJson.getBytes(UTF_8));
    return op;
  } catch (TException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public @Nullable <T> T decodeBody(@NotNull ByteBuf arg3, @NotNull Class<T> bodyType) {

    try {
        // Create a new instance of type 'T'
        T base = bodyType.getConstructor().newInstance();

        // Get byte[] from ByteBuf
        byte[] payloadBytes = new byte[arg3.readableBytes()];
        arg3.readBytes(payloadBytes);

        // Actually deserialize the payload
        TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
        deserializer.deserialize((TBase<?, ?>) base, payloadBytes);

        return base;
    } catch (
        NoSuchMethodException
            | InvocationTargetException
            | InstantiationException
            | IllegalAccessException
            | TException e
    ) {
        logger.error("Failed to decode body to {}", bodyType.getName(), e);
    }

    return null;

}
 

@Override
public JobSpecT getJobSpec(UUID jobId) {

    final String json = jdbc.queryForObject("SELECT str_thrift_spec FROM job_history WHERE pk_job=?",
            String.class, jobId);

    final TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
    final JobSpecT spec = new JobSpecT();
    try {
        deserializer.deserialize(spec, json.getBytes());
        return spec;
    } catch (TException e) {
        throw new JobSpecException("Failed to parse job spec " + e, e);
    }
}
 

@SuppressWarnings("rawtypes")
public TBase decodeMessage(String topic, byte[] payload)
        throws InstantiationException, IllegalAccessException, TException {
    TDeserializer serializer = new TDeserializer(messageProtocolFactory);
    TBase result = this.getMessageClass(topic).newInstance();
    serializer.deserialize(result, payload);
    return result;
}
 

@SuppressWarnings("rawtypes")
public static TBase deserialize(TBase baseObject, byte[] serialized) throws IOException {
  TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory(maxMessageSize, maxMessageSize));
  try {
    deserializer.deserialize(baseObject, serialized);
  } catch (TException e) {
    throw new IOException("Error deserializing thrift object "
        + baseObject, e);
  }
  return baseObject;
}
 

public static TBase toEvent(byte[] data, Class<? extends TBase> clazz) {
	TBase tBase;
	try {
		tBase = clazz.newInstance();
		TDeserializer tDeserializer = tDeserializerLocal.get();
		tDeserializer.deserialize(tBase, data);
	} catch (Exception e) {
		throw new RuntimeException("fail to deserialize into thrift object of type " + clazz, e);
	}

	return tBase;
}
 

/**
 * Decodes a binary-encoded byte array into a target type.
 *
 * @param clazz Class to instantiate and deserialize to.
 * @param buffer Buffer to decode.
 * @param <T> Target type.
 * @return A populated message.
 * @throws CodingException If the message could not be decoded.
 */
public static <T extends TBase<T, ?>> T decodeNonNull(Class<T> clazz, byte[] buffer)
    throws CodingException {

  requireNonNull(clazz);
  requireNonNull(buffer);

  try {
    T t = newInstance(clazz);
    new TDeserializer(PROTOCOL_FACTORY).deserialize(t, buffer);
    return t;
  } catch (TException e) {
    throw new CodingException("Failed to deserialize thrift object.", e);
  }
}
 

public void run() {
  stopped = false;
  KafkaConsumer<byte[], byte[]> kafkaConsumer = null;
  try {
    // read audit messages from kafka
    Properties properties = SingerTestHelper.createKafkaConsumerConfig();

    kafkaConsumer = new KafkaConsumer<>(properties);
    kafkaConsumer.subscribe(Arrays.asList(auditTopic));

    String hostName = SingerUtils.getHostname();
    System.out.println("Fetching auditing messages from " + hostName + " : ");
    TDeserializer deserializer = new TDeserializer();
    while (!stopped) {
      ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(Duration.ofMillis(500));
      for (ConsumerRecord<byte[], byte[]> record : records) {
        byte[] bytes = record.value();
        AuditMessage auditMessage = new AuditMessage();
        deserializer.deserialize(auditMessage, bytes);
        if (auditMessage.hostname.equals(hostName) &&
            auditMessage.timestamp >= startTimestampInMillis) {
          this.auditCount += auditMessage.numMessages;
          System.out.println("num messages = " + auditCount + "  " + auditMessage);
        }
      }
    }
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    if (kafkaConsumer != null) {
      kafkaConsumer.close();
    }
  }
}
 

public static void deserialize(ThriftProtocol protocol, byte[] source, TBase<?, ?> dest)
    throws ThriftException {
  TDeserializer deserializer = new TDeserializer(getProtocolFactory(protocol));
  dest.clear();
  try {
    deserializer.deserialize(dest, source);
  } catch (TException e) {
    throw new ThriftException(e);
  }
}
 

/**
 * Reads a file in and runs a predicate on each deserialized rule key
 *
 * @param filename The name of the file to read
 * @param visitor Called for each rule key that gets deserialized. Return {@code true} if
 *     deserialization should halt, or false if it should proceed
 * @throws ParseException There was an error reading data, or invalid data was found in the file
 */
public void readFile(Path filename, Predicate<FullRuleKey> visitor) throws ParseException {
  ByteBuffer lengthBuf = ByteBuffer.allocate(4);
  try (FileInputStream fileInputStream = new FileInputStream(filename.toFile())) {
    while (fileInputStream.available() >= 4) {
      fileInputStream.read(lengthBuf.array());
      int length = lengthBuf.getInt();
      lengthBuf.rewind();

      byte[] serialized = new byte[length];
      int bytesRead = fileInputStream.read(serialized);
      if (bytesRead != length) {
        throw new ParseException(
            filename,
            "Invalid length specified. Expected %s bytes, only got %s",
            length,
            bytesRead);
      }
      TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
      FullRuleKey ruleKey = new FullRuleKey();
      deserializer.deserialize(ruleKey, serialized);
      // The thrift deserializer doesn't blow up on invalid data, it just sets all fields to
      // null. 'key' is required, so if it's null, we failed to deserialize. Yes, deserialize()
      // /should/ throw a TException, but it doesn't.
      if (ruleKey.key == null) {
        throw new ParseException(
            filename, "Could not deserialize array of size %s", serialized.length);
      }
      if (visitor.test(ruleKey)) {
        return;
      }
    }
  } catch (TException | IOException e) {
    throw new ParseException(e, filename, "Error reading file: %s", e.getMessage());
  }
}
 

@Test
public void testWritesToGivenFileAndCreatesDirectories() throws Exception {
  Path logPath = logDir.resolve("logs").resolve("out.log").toAbsolutePath();
  try (ThriftRuleKeyLogger logger = ThriftRuleKeyLogger.create(logPath)) {
    Assert.assertNotNull(logger);
    logger.write(
        new FullRuleKey(
            "rule_key", "name", "rule_type", ImmutableMap.of("key", Value.stringValue("value"))));
  }
  Assert.assertTrue(logPath.toFile().exists());

  ByteBuffer lengthBuf = ByteBuffer.allocate(4);
  try (FileInputStream logFileStream = new FileInputStream(logPath.toFile())) {
    Assert.assertTrue(logFileStream.available() > 4);

    logFileStream.read(lengthBuf.array());
    int length = lengthBuf.getInt();

    Assert.assertEquals(length, logFileStream.available()); // Only should have one object

    byte[] serialized = new byte[length];
    logFileStream.read(serialized);
    TDeserializer serializer = new TDeserializer(new TCompactProtocol.Factory());
    FullRuleKey ruleKey = new FullRuleKey();
    serializer.deserialize(ruleKey, serialized);

    Assert.assertEquals("rule_key", ruleKey.key);
    Assert.assertEquals("name", ruleKey.name);
    Assert.assertEquals("rule_type", ruleKey.type);
    Assert.assertEquals(1, ruleKey.values.size());
    Assert.assertEquals("value", ruleKey.values.get("key").getStringValue());
  }
}
 

private FullRuleKey getRuleKey() throws TException {
  byte[] outputArray = output.toByteArray();
  ByteBuffer lengthBuf = ByteBuffer.wrap(outputArray, 0, 4);
  int length = lengthBuf.getInt();

  Assert.assertEquals(outputArray.length - 4, length);

  TDeserializer serializer = new TDeserializer(new TCompactProtocol.Factory());
  FullRuleKey ruleKey = new FullRuleKey();
  serializer.deserialize(ruleKey, Arrays.copyOfRange(outputArray, 4, outputArray.length));
  return ruleKey;
}
 

public static <T> T thriftDeserialize(Class c, byte[] b, int offset, int length) {
    try {
        T ret = (T) c.newInstance();
        TDeserializer des = getDes();
        des.deserialize((TBase) ret, b, offset, length);
        return ret;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public <T> T deserialize(byte[] bytes, Class<T> clazz) {
    try {
        TBase instance = (TBase) clazz.newInstance();
        new TDeserializer().deserialize(instance, Utils.gunzip(bytes));
        return (T) instance;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public <T> T deserialize(byte[] bytes, Class<T> clazz) {
    try {
        TBase instance = (TBase) clazz.newInstance();
        new TDeserializer().deserialize(instance, bytes);
        return (T) instance;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

public static <T> T thriftDeserialize(Class c, byte[] b, int offset, int length) {
    try {
        T ret = (T) c.newInstance();
        TDeserializer des = getDes();
        des.deserialize((TBase) ret, b, offset, length);
        return ret;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private static TBase deserializeBytesToThriftObj(byte[] bytes, String thriftSchema)
    throws Exception {
  TBase obj = (TBase) Class.forName(thriftSchema).getConstructor().newInstance();
  TDeserializer deserializer = new TDeserializer();
  deserializer.deserialize(obj, bytes);
  return obj;
}
 

public Transaction decode(byte[] encoded) throws IOException {
  TTransaction thriftTx = new TTransaction();
  TDeserializer deserializer = new TDeserializer();
  try {
    deserializer.deserialize(thriftTx, encoded);
    return TransactionConverterUtils.unwrap(thriftTx);
  } catch (TException te) {
    throw new IOException(te);
  }
}
 

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
  Object top = stack.pop();
  
  if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof GTSEncoder)) {
    throw new WarpScriptException(getName() + " operates on a string, byte array or encoder.");
  }
  
  Map<String,GTSEncoder> encoders = new HashMap<String,GTSEncoder>();
  
  GTSDecoder decoder;
  
  if (top instanceof GTSEncoder) {
    decoder = ((GTSEncoder) top).getUnsafeDecoder(false);
  } else {
    try {
      byte[] bytes = top instanceof String ? OrderPreservingBase64.decode(top.toString().getBytes(StandardCharsets.US_ASCII)) : (byte[]) top;
      
      TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
      
      GTSWrapper wrapper = new GTSWrapper();
      
      deser.deserialize(wrapper, bytes);

      decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);        
    } catch (TException te) {
      throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
    }            
  }

  GTSEncoder enc;
  
  try {
    while(decoder.next()) {
      Object value = decoder.getBinaryValue();
      
      String type = "DOUBLE";
      
      if (value instanceof String) {
        type = "STRING";
      } else if (value instanceof Boolean) {
        type = "BOOLEAN";
      } else if (value instanceof Long) {
        type = "LONG";
      } else if (value instanceof Double || value instanceof BigDecimal) { 
        type = "DOUBLE";
      } else if (value instanceof byte[]) {
        type = "BINARY";
      }
      
      enc = encoders.get(type);
      
      if (null == enc) {
        enc = new GTSEncoder(0L);
        enc.setMetadata(decoder.getMetadata());
        encoders.put(type, enc);
      }

      enc.addValue(decoder.getTimestamp(), decoder.getLocation(), decoder.getElevation(), value);
    }      
  } catch (Exception e) {
    throw new WarpScriptException(getName() + " encountered an exception during conversion.");
  }
      
  stack.push(encoders);

  return stack;
}
 

@Override
protected TDeserializer initialValue() {
  return new TDeserializer(new TBinaryProtocol.Factory());
}
 

@Override
protected TDeserializer initialValue() {
  return new TDeserializer(new TCompactProtocol.Factory());
}