Java Code Examples for org.apache.thrift.protocol.TCompactProtocol#Factory

The following examples show how to use org.apache.thrift.protocol.TCompactProtocol#Factory . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TOMVSTRING.java    From warp10-platform with Apache License 2.0 6 votes vote down vote up
private static StringBuilder bytesToString(StringBuilder sb, byte[] bytes) {
  TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
  GTSWrapper wrapper = new GTSWrapper();

  try {
    deser.deserialize(wrapper, bytes);

    return wrapperToString(sb, wrapper);
  } catch (TException te) {
    if (null == sb) {
      sb = new StringBuilder();
    }
    sb.append("b64:");
    sb.append(Base64.encodeBase64URLSafeString(bytes));
    return sb;
  }    
}
 
Example 2
Source File: LogUtil.java    From warp10-platform with Apache License 2.0 6 votes vote down vote up
public static final String serializeLoggingEvent(KeyStore keystore, LoggingEvent event) {
  if (null == event) {
    return null;
  }
  
  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  byte[] serialized = null;
  
  try {
    serialized = serializer.serialize(event);
  } catch (TException te) {
    return null;
  }
  
  if (!checkedAESKey) {
    checkedAESKey = true;
    loggingAESKey = keystore.getKey(KeyStore.AES_LOGGING);      
  }
  if (null != loggingAESKey) {
    serialized = CryptoUtils.wrap(loggingAESKey, serialized);
  }
  
  return new String(OrderPreservingBase64.encode(serialized), StandardCharsets.US_ASCII);
}
 
Example 3
Source File: TestThriftToParquetFileWriter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private <T extends TBase<?,?>> Path createFile(T... tObjs) throws IOException, InterruptedException, TException  {
  final Path fileToCreate = new Path("target/test/TestThriftToParquetFileWriter/"+tObjs[0].getClass()+".parquet");
  LOG.info("File created: {}", fileToCreate.toString());
  Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) {
    fs.delete(fileToCreate, true);

  }
  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, (Class<? extends TBase<?, ?>>) tObjs[0].getClass());

  for(T tObj:tObjs) {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

    tObj.write(protocol);

    w.write(new BytesWritable(baos.toByteArray()));
  }
  w.close();

  return fileToCreate;
}
 
Example 4
Source File: ThriftProtocolFactories.java    From armeria with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the {@link SerializationFormat} for the specified {@link TProtocolFactory}.
 *
 * @throws IllegalArgumentException if the specified {@link TProtocolFactory} is not known by this class
 */
public static SerializationFormat toSerializationFormat(TProtocolFactory protoFactory) {
    requireNonNull(protoFactory, "protoFactory");

    if (protoFactory instanceof TBinaryProtocol.Factory) {
        return ThriftSerializationFormats.BINARY;
    } else if (protoFactory instanceof TCompactProtocol.Factory) {
        return ThriftSerializationFormats.COMPACT;
    } else if (protoFactory instanceof TJSONProtocol.Factory) {
        return ThriftSerializationFormats.JSON;
    } else if (protoFactory instanceof TTextProtocolFactory) {
        final TTextProtocolFactory factory = (TTextProtocolFactory) protoFactory;
        return factory.usesNamedEnums() ? ThriftSerializationFormats.TEXT_NAMED_ENUM
                                        : ThriftSerializationFormats.TEXT;
    } else {
        throw new IllegalArgumentException(
                "unsupported TProtocolFactory: " + protoFactory.getClass().getName());
    }
}
 
Example 5
Source File: DKGraphxServer.java    From dk-fitting with Apache License 2.0 5 votes vote down vote up
/**
 * 阻塞式、多线程处理
 *
 * @param args
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) {
    try {
        int port = Integer.valueOf(prop.get("dkgraphx.port"));
        //设置传输通道,普通通道
        TServerTransport serverTransport = new TServerSocket(port);

        //使用高密度二进制协议
        TProtocolFactory proFactory = new TCompactProtocol.Factory();

        //设置处理器
        TProcessor processor = new DKGraphx.Processor(new DKGraphxImpl());

        //创建服务器
        TServer server = new TThreadPoolServer(
                new TThreadPoolServer.Args(serverTransport)
                        .protocolFactory(proFactory)
                        .processor(processor)
        );

        System.out.println("Start server on port "+ port +"...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 6
Source File: CommandHeaderTBaseSerializerFactory.java    From pinpoint with Apache License 2.0 5 votes vote down vote up
public CommandHeaderTBaseSerializerFactory(int outputStreamSize) {
    TypeLocator<TBase<?, ?>> commandTbaseLocator = TCommandRegistry.build(Arrays.asList(TCommandType.values()));

    TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
    HeaderTBaseSerializerFactory serializerFactory = new HeaderTBaseSerializerFactory(true, outputStreamSize, protocolFactory, commandTbaseLocator);
    this.factory = new ThreadLocalHeaderTBaseSerializerFactory<HeaderTBaseSerializer>(serializerFactory);
}
 
Example 7
Source File: GeneralFactory.java    From ikasoa with MIT License 5 votes vote down vote up
/**
 * 获取客户端AsyncService对象
 */
@Override
public AsyncService getAsyncService(TNonblockingTransport transport, String serviceName) throws IkasoaException {
	if (ObjectUtil.isNull(transport))
		throw new IllegalArgumentException("'transport' is null !");
	try {
		return StringUtil.isEmpty(serviceName)
				? new AsyncServiceClientImpl((TProtocolFactory) new TCompactProtocol.Factory(), transport)
				: new AsyncServiceClientImpl(new AsyncMultiplexedProtocolFactory(serviceName), transport);
	} catch (IOException e) {
		throw new IkasoaException(e);
	}
}
 
Example 8
Source File: ThriftSerializer.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
public static TBase deserialize(TBase baseObject, byte[] serialized) throws IOException {
  TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory(maxMessageSize, maxMessageSize));
  try {
    deserializer.deserialize(baseObject, serialized);
  } catch (TException e) {
    throw new IOException("Error deserializing thrift object "
        + baseObject, e);
  }
  return baseObject;
}
 
Example 9
Source File: TestParquetToThriftReadWriteAndProjection.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private <T extends TBase<?, ?>> void shouldDoProjection(Configuration conf, T recordToWrite, T exptectedReadResult, Class<? extends TBase<?, ?>> thriftClass) throws Exception {
  final Path parquetFile = new Path("target/test/TestParquetToThriftReadWriteAndProjection/file.parquet");
  final FileSystem fs = parquetFile.getFileSystem(conf);
  if (fs.exists(parquetFile)) {
    fs.delete(parquetFile, true);
  }

  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, thriftClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  recordToWrite.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();


  final ParquetThriftInputFormat<T> parquetThriftInputFormat = new ParquetThriftInputFormat<T>();
  final Job job = new Job(conf, "read");
  job.setInputFormatClass(ParquetThriftInputFormat.class);
  ParquetThriftInputFormat.setInputPaths(job, parquetFile);
  final JobID jobID = new JobID("local", 1);
  List<InputSplit> splits = parquetThriftInputFormat.getSplits(ContextUtil.newJobContext(ContextUtil.getConfiguration(job), jobID));
  T readValue = null;
  for (InputSplit split : splits) {
    TaskAttemptContext taskAttemptContext = ContextUtil.newTaskAttemptContext(ContextUtil.getConfiguration(job), new TaskAttemptID(new TaskID(jobID, true, 1), 0));
    final RecordReader<Void, T> reader = parquetThriftInputFormat.createRecordReader(split, taskAttemptContext);
    reader.initialize(split, taskAttemptContext);
    if (reader.nextKeyValue()) {
      readValue = reader.getCurrentValue();
      LOG.info("{}", readValue);
    }
  }
  assertEquals(exptectedReadResult, readValue);

}
 
Example 10
Source File: ParquetScroogeSchemeTest.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void writeParquetFile(List<TBase> recordsToWrite, Configuration conf, Path parquetFile) throws IOException, InterruptedException, org.apache.thrift.TException {
  //create a test file
  final TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  final TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  Class writeClass = recordsToWrite.get(0).getClass();
  final ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(parquetFile, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, writeClass);
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  for (TBase recordToWrite : recordsToWrite) {
    recordToWrite.write(protocol);
  }
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 
Example 11
Source File: TestParquetTBaseScheme.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void createFileForRead() throws Exception {
  final Path fileToCreate = new Path(parquetInputPath+"/names.parquet");

  final Configuration conf = new Configuration();
  final FileSystem fs = fileToCreate.getFileSystem(conf);
  if (fs.exists(fileToCreate)) fs.delete(fileToCreate, true);

  TProtocolFactory protocolFactory = new TCompactProtocol.Factory();
  TaskAttemptID taskId = new TaskAttemptID("local", 0, true, 0, 0);
  ThriftToParquetFileWriter w = new ThriftToParquetFileWriter(fileToCreate, ContextUtil.newTaskAttemptContext(conf, taskId), protocolFactory, Name.class);

  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));

  Name n1 = new Name();
  n1.setFirst_name("Alice");
  n1.setLast_name("Practice");
  Name n2 = new Name();
  n2.setFirst_name("Bob");
  n2.setLast_name("Hope");
  Name n3 = new Name();
  n3.setFirst_name("Charlie");
  n3.setLast_name("Horse");

  n1.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n2.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  baos.reset();
  n3.write(protocol);
  w.write(new BytesWritable(baos.toByteArray()));
  w.close();
}
 
Example 12
Source File: ThriftRuleKeyLoggerTest.java    From buck with Apache License 2.0 5 votes vote down vote up
@Test
public void testWritesToGivenFileAndCreatesDirectories() throws Exception {
  Path logPath = logDir.resolve("logs").resolve("out.log").toAbsolutePath();
  try (ThriftRuleKeyLogger logger = ThriftRuleKeyLogger.create(logPath)) {
    Assert.assertNotNull(logger);
    logger.write(
        new FullRuleKey(
            "rule_key", "name", "rule_type", ImmutableMap.of("key", Value.stringValue("value"))));
  }
  Assert.assertTrue(logPath.toFile().exists());

  ByteBuffer lengthBuf = ByteBuffer.allocate(4);
  try (FileInputStream logFileStream = new FileInputStream(logPath.toFile())) {
    Assert.assertTrue(logFileStream.available() > 4);

    logFileStream.read(lengthBuf.array());
    int length = lengthBuf.getInt();

    Assert.assertEquals(length, logFileStream.available()); // Only should have one object

    byte[] serialized = new byte[length];
    logFileStream.read(serialized);
    TDeserializer serializer = new TDeserializer(new TCompactProtocol.Factory());
    FullRuleKey ruleKey = new FullRuleKey();
    serializer.deserialize(ruleKey, serialized);

    Assert.assertEquals("rule_key", ruleKey.key);
    Assert.assertEquals("name", ruleKey.name);
    Assert.assertEquals("rule_type", ruleKey.type);
    Assert.assertEquals(1, ruleKey.values.size());
    Assert.assertEquals("value", ruleKey.values.get("key").getStringValue());
  }
}
 
Example 13
Source File: BrokerRequestSerializationTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
@Test
  public static void testSerialization()
      throws TException {
    BrokerRequest req = new BrokerRequest();

    // Populate Query Type
    QueryType type = new QueryType();
    type.setHasAggregation(true);
    type.setHasFilter(true);
    type.setHasSelection(true);
    type.setHasGroup_by(true);
    req.setQueryType(type);

    // Populate Query source
    QuerySource s = new QuerySource();
    s.setTableName("dummy");
    req.setQuerySource(s);

    req.setDuration("dummy");
    req.setTimeInterval("dummy");

    //Populate Group-By
    GroupBy groupBy = new GroupBy();
    List<String> columns = new ArrayList<String>();
    columns.add("dummy1");
    columns.add("dummy2");
    groupBy.setColumns(columns);
    groupBy.setTopN(100);
    req.setGroupBy(groupBy);

    //Populate Selections
    Selection sel = new Selection();
    sel.setSize(1);
    SelectionSort s2 = new SelectionSort();
    s2.setColumn("dummy1");
    s2.setIsAsc(true);
    sel.addToSelectionSortSequence(s2);
    sel.addToSelectionColumns("dummy1");
    req.setSelections(sel);

    //Populate FilterQuery
    FilterQuery q1 = new FilterQuery();
    q1.setId(1);
    q1.setColumn("dummy1");
    q1.addToValue("dummy1");
    q1.addToNestedFilterQueryIds(2);
    q1.setOperator(FilterOperator.AND);
    FilterQuery q2 = new FilterQuery();
    q2.setId(2);
    q2.setColumn("dummy2");
    q2.addToValue("dummy2");
    q2.setOperator(FilterOperator.AND);

    FilterQueryMap map = new FilterQueryMap();
    map.putToFilterQueryMap(1, q1);
    map.putToFilterQueryMap(2, q2);
    req.setFilterQuery(q1);
    req.setFilterSubQueryMap(map);

    //Populate Aggregations
    AggregationInfo agg = new AggregationInfo();
    agg.setAggregationType("dummy1");
    agg.putToAggregationParams("key1", "dummy1");
    req.addToAggregationsInfo(agg);

    TSerializer normalSerializer = new TSerializer();
    TSerializer compactSerializer = new TSerializer(new TCompactProtocol.Factory());
    normalSerializer.serialize(req);
    compactSerializer.serialize(req);

//    int numRequests = 100000;
//    TimerContext t = MetricsHelper.startTimer();
//    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
//    //TSerializer serializer = new TSerializer();
//    //Compact : Size 183 , Serialization Latency : 0.03361ms
//    // Normal : Size 385 , Serialization Latency : 0.01144ms
//
//    for (int i = 0; i < numRequests; i++) {
//      try {
//        serializer.serialize(req);
//        //System.out.println(s3.length);
//        //break;
//      } catch (TException e) {
//        e.printStackTrace();
//      }
//    }
//    t.stop();
//    System.out.println("Latency is :" + (t.getLatencyMs() / (float) numRequests));
  }
 
Example 14
Source File: StandaloneChunkedMemoryStore.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
private void load(String path) throws IOException {
  
  long nano = System.nanoTime();
  long chunks = 0;
  long datapoints = 0;
  long bytes = 0L;
  
  Configuration conf = new Configuration();
      
  conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
  conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
  
  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  
  TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
  
  SequenceFile.Reader.Option optPath = SequenceFile.Reader.file(new Path(path));    
  
  SequenceFile.Reader reader = null;
  
  boolean failsafe = "true".equals(properties.getProperty(io.warp10.continuum.Configuration.STANDALONE_MEMORY_STORE_LOAD_FAILSAFE));
  
  try {
    reader = new SequenceFile.Reader(conf, optPath);

    System.out.println("Loading '" + path + "' back in memory.");

    while(reader.next(key, value)) {
      chunks++;
      GTSWrapper wrapper = new GTSWrapper();
      deserializer.deserialize(wrapper, key.copyBytes());
      GTSEncoder encoder = new GTSEncoder(wrapper.getBase(), null, value.copyBytes());
      encoder.setCount(wrapper.getCount());
      datapoints += wrapper.getCount();
      bytes += value.getLength() + key.getLength();
      if (wrapper.isSetMetadata()) {
        encoder.safeSetMetadata(wrapper.getMetadata());
      } else {
        encoder.safeSetMetadata(new Metadata());
      }
      store(encoder);
      if (null != this.directoryClient) {
        this.directoryClient.register(encoder.getMetadata());
      }
    }
  } catch (FileNotFoundException fnfe) {
    System.err.println("File '" + path + "' was not found, skipping.");
    return;
  } catch (IOException ioe) {
    if (!failsafe) {
      throw ioe;
    } else {
      System.err.println("Ignoring exception " + ioe.getMessage() + ".");
    }
  } catch (Exception e) {
    if (!failsafe) {
      throw new IOException(e);
    } else {
      System.err.println("Ignoring exception " + e.getMessage() + ".");
    }
  }
  
  reader.close();    
  
  nano = System.nanoTime() - nano;
  
  System.out.println("Loaded " + chunks + " chunks (" + datapoints + " datapoints, " + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
}
 
Example 15
Source File: UNSECURE.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
  
  Object o = stack.pop();

  if (this.checkkey && null == stack.getAttribute(WarpScriptStack.ATTRIBUTE_SECURE_KEY)) {
    throw new WarpScriptException("You need to set the secure key first.");
  }
  
  if (!(o instanceof String)) {
    throw new WarpScriptException(getName() + " operates on a string.");
  }
  
  // Retrieve raw bytes
  byte[] raw = OrderPreservingBase64.decode(o.toString().getBytes(StandardCharsets.US_ASCII));
  
  // Unwrap
  
  synchronized(SECURE.class) {
    if (null == aesKey) {
      try {
        aesKey = WarpDist.getKeyStore().getKey(KeyStore.AES_SECURESCRIPTS);
      } catch (Throwable t) {
        // Catch NoClassDefFoundError
      }
    }
  }
  
  if (null == aesKey) {
    throw new WarpScriptException("Missing secure script encryption key.");
  }
  
  byte[] unwrapped = CryptoUtils.unwrap(aesKey, raw);

  // Deserialize
  TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
  
  SecureScript sscript = new SecureScript();
  
  try {
    deserializer.deserialize(sscript, unwrapped);
  } catch (TException te) {
    throw new WarpScriptException("Unable to unsecure script.", te);
  }
  
  if (this.checkkey) {
    if (!stack.getAttribute(WarpScriptStack.ATTRIBUTE_SECURE_KEY).toString().equals(sscript.getKey())) {
      throw new WarpScriptException("Invalid secure key.");
    }
  }
  
  // Decompress script content if needed
  
  if (sscript.isCompressed()) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ByteArrayInputStream bais = new ByteArrayInputStream(sscript.getScript());
    
    try {
      GZIPInputStream gzipin = new GZIPInputStream(bais);
      
      byte[] buf = new byte[128];
      
      while(true) {
        int len = gzipin.read(buf);
        if (len < 0) {
          break;
        }
        baos.write(buf, 0, len);
      }
      
      sscript.setCompressed(false);
      sscript.setScript(baos.toByteArray());        
    } catch (IOException ioe) {
      throw new WarpScriptException("Unable to unsecure script.", ioe);
    }
  }
  
  // Convert bytes to String
  String script = new String(sscript.getScript(), StandardCharsets.UTF_8);

  stack.push(script);
  
  return stack;
}
 
Example 16
Source File: Directory.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
void handleStats(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { 
  if (!Constants.API_ENDPOINT_DIRECTORY_STATS_INTERNAL.equals(target)) {
    return;
  }
  
  long nano = System.nanoTime();

  baseRequest.setHandled(true);

  //
  // Read DirectoryRequests from stdin
  //
  
  BufferedReader br = new BufferedReader(request.getReader());
  
  while (true) {
    String line = br.readLine();
    
    if (null == line) {
      break;
    }
    
    byte[] raw = OrderPreservingBase64.decode(line.getBytes(StandardCharsets.US_ASCII));

    // Extract DirectoryStatsRequest
    TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
    DirectoryStatsRequest req = new DirectoryStatsRequest();
    
    try {
      deser.deserialize(req, raw);
      DirectoryStatsResponse resp = stats(req);

      response.setContentType("text/plain");
      OutputStream out = response.getOutputStream();
            
      TSerializer ser = new TSerializer(new TCompactProtocol.Factory());
      byte[] data = ser.serialize(resp);
      
      OrderPreservingBase64.encodeToStream(data, out);
      
      out.write('\r');
      out.write('\n');
    } catch (TException te) {
      throw new IOException(te);
    }            
  }
}
 
Example 17
Source File: UNWRAP.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
  Object top = stack.pop();
  
  if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof List)) {
    throw new WarpScriptException(getName() + " operates on a string or byte array or a list thereof.");
  }
  
  List<Object> inputs = new ArrayList<Object>();
  
  if (top instanceof String || top instanceof byte[]) {
    inputs.add(top);
  } else {
    for (Object o: (List) top) {
      if (!(o instanceof String) && !(o instanceof byte[])) {
        throw new WarpScriptException(getName() + " operates on a string or byte array or a list thereof.");
      }
      inputs.add(o);
    }
  }
  
  List<Object> outputs = new ArrayList<Object>();
  
  for (Object s: inputs) {
    byte[] bytes = s instanceof String ? OrderPreservingBase64.decode(s.toString().getBytes(StandardCharsets.US_ASCII)) : (byte[]) s;
    
    TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());
    
    try {
      GTSWrapper wrapper = new GTSWrapper();
      
      deser.deserialize(wrapper, bytes);
      
      GeoTimeSerie gts = GTSWrapperHelper.fromGTSWrapperToGTS(wrapper, this.empty);
      
      outputs.add(gts);
    } catch (TException te) {
      throw new WarpScriptException(getName() + " failed to unwrap GTS.", te);
    }      
  }
  
  if (!(top instanceof List)) {
    stack.push(outputs.get(0));      
  } else {
    stack.push(outputs);
  }
  
  return stack;
}
 
Example 18
Source File: DemoClientTraditionalTEST.java    From nettythrift with Apache License 2.0 4 votes vote down vote up
@Test
public void test_AsyncClient() throws Throwable {
	Random rnd = new Random(System.nanoTime());

	TProtocolFactory[] protfacs = new TProtocolFactory[] { new TCompactProtocol.Factory(),
			new TBinaryProtocol.Factory(), new TJSONProtocol.Factory(),
			new TSimpleJSONProtocol.Factory(TCalculator.Iface.class, false) };

	TProtocolFactory protocolFactory = protfacs[rnd.nextInt(protfacs.length)];

	System.out.println("protocolFactory: " + protocolFactory);

	TAsyncClientManager clientManager = new TAsyncClientManager();
	TNonblockingTransport transport = new TNonblockingSocket(HOST, PORT);
	TCalculator.AsyncClient client = new TCalculator.AsyncClient(protocolFactory, clientManager, transport);
	final int num1 = rnd.nextInt(Integer.MAX_VALUE / 2 - 1);
	final int num2 = rnd.nextInt(Integer.MAX_VALUE / 2 - 1);

	final CountDownLatch latch = new CountDownLatch(1);
	final Throwable[] exceptions = new Throwable[1];
	AsyncMethodCallback<TCalculator.AsyncClient.add_call> resultHandler = new AsyncMethodCallback<TCalculator.AsyncClient.add_call>() {
		@Override
		public void onComplete(TCalculator.AsyncClient.add_call response) {
			System.out.println("onComplete!");
			try {
				int result = response.getResult();
				Assert.assertEquals(num1 + num2, result);
			} catch (Throwable e) {
				exceptions[0] = e;
			} finally {
				latch.countDown();
			}
		}

		@Override
		public void onError(Exception exception) {
			System.err.println("onError!");
			exception.printStackTrace();
			latch.countDown();
		}

	};
	client.add(num1, num2, resultHandler);
	latch.await();
	transport.close();
	if (exceptions[0] != null) {
		throw exceptions[0];
	}
}
 
Example 19
Source File: StandaloneMemoryStore.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
private void load(String path) throws IOException {
  
  long nano = System.nanoTime();
  int gts = 0;
  long bytes = 0L;
  
  Configuration conf = new Configuration();
      
  conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
  conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  
  TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
  
  SequenceFile.Reader.Option optPath = SequenceFile.Reader.file(new Path(path));    
  
  SequenceFile.Reader reader = null;
  
  try {
    reader = new SequenceFile.Reader(conf, optPath);

    System.out.println("Loading '" + path + "' back in memory.");

    while(reader.next(key, value)) {
      gts++;
      GTSWrapper wrapper = new GTSWrapper();
      deserializer.deserialize(wrapper, key.copyBytes());
      GTSEncoder encoder = new GTSEncoder(0L, null, value.copyBytes());
      encoder.setCount(wrapper.getCount());
      
      bytes += value.getLength() + key.getLength();
      if (wrapper.isSetMetadata()) {
        encoder.safeSetMetadata(wrapper.getMetadata());
      } else {
        encoder.safeSetMetadata(new Metadata());
      }
      store(encoder);
      if (null != this.directoryClient) {
        this.directoryClient.register(encoder.getMetadata());
      }
    }
  } catch (FileNotFoundException fnfe) {
    System.err.println("File '" + path + "' was not found, skipping.");
    return;
  } catch (IOException ioe) {
    throw ioe;
  } catch (Exception e) {
    throw new IOException(e);
  }
  
  reader.close();    
  
  nano = System.nanoTime() - nano;
  
  System.out.println("Loaded " + gts + " GTS (" + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
}
 
Example 20
Source File: ThriftCodec.java    From singer with Apache License 2.0 4 votes vote down vote up
@Override
protected TSerializer initialValue() {
  return new TSerializer(new TCompactProtocol.Factory());
}