Java Code Examples for scala.collection.JavaConversions#asJavaIterable()

public static void refresh() {
    if (null == CONTEXT)
        throw new IllegalStateException("The Spark context has not been created.");
    if (CONTEXT.isStopped())
        recreateStopped();

    final Set<String> keepNames = new HashSet<>();
    for (final RDD<?> rdd : JavaConversions.asJavaIterable(CONTEXT.persistentRdds().values())) {
        if (null != rdd.name()) {
            keepNames.add(rdd.name());
            NAME_TO_RDD.put(rdd.name(), rdd);
        }
    }
    // remove all stale names in the NAME_TO_RDD map
    NAME_TO_RDD.keySet().stream().filter(key -> !keepNames.contains(key)).collect(Collectors.toList()).forEach(NAME_TO_RDD::remove);
}
 

public static StreamSource source(SourceConf sc){
 StreamSource s = new StreamSource();
 s.setAlias(sc.alias());
 s.setSqlQuery(sc.query());
 if (sc.slide().isDefined())
  s.setRawSlideValue(sc.slide().get());
 if (sc.samplingRate().isDefined())
  s.setSamplingRate(((Double)sc.samplingRate().get()).floatValue());
 if (sc.disconnectBufferSize().isDefined())
  s.setDisconnectedBufferSize(((Integer)sc.disconnectBufferSize().get()));
 if (sc.storageSize().isDefined())
  s.setRawHistorySize(sc.storageSize().get());
 AddressBean[] add=new AddressBean[sc.wrappers().size()];
 int i=0;
 for (WrapperConf w:JavaConversions.asJavaIterable(sc.wrappers())){
  add[i]=address(w);
  i++;
 }
 s.setAddressing(add);
 return s;
}
 

public static AddressBean address(WrapperConf w){
    KeyValueImp [] p=new KeyValueImp[w.params().size()];
    Iterable<String> keys=JavaConversions.asJavaIterable(w.params().keys());
    int i=0;
 for (String k:keys){
  p[i]=new KeyValueImp(k,w.params().apply(k));
  i++;
 }
    AddressBean a = new AddressBean(w.wrapper(),p);
    if(w.partialKey().isDefined()){
    a.setPartialOrderKey(w.partialKey().get());
    }
    DataField [] out=new DataField[(w.output().size())];
 for (int j=0;j<out.length;j++){
  out[j]=dataField(w.output().apply(j));
 }
    a.setVsconfig(out);
 return a;
}
 

/**
 * There is always only one string in zkHost
 * @param zkHost
 * @return
 */
public static Set<String> getBrokers(Set<String> zkHost)
{
  ZkClient zkclient = new ZkClient(zkHost.iterator().next(), 30000, 30000, ZKStringSerializer$.MODULE$);
  Set<String> brokerHosts = new HashSet<String>();
  for (Broker b : JavaConversions.asJavaIterable(ZkUtils.getAllBrokersInCluster(zkclient))) {
    brokerHosts.add(b.connectionString());
  }
  zkclient.close();
  return brokerHosts;
}
 

private static boolean hasPersistedRDD(final String name) {
    for (final RDD<?> rdd : JavaConversions.asJavaIterable(Spark.getContext().persistentRdds().values())) {
        if (null != rdd.name() && rdd.name().equals(name))
            return true;
    }
    return false;
}
 

private static int getPersistedRDDSize() {
    int counter = 0;
    for (final RDD<?> rdd : JavaConversions.asJavaIterable(Spark.getContext().persistentRdds().values())) {
        if (null != rdd.name())
            counter++;
    }
    return counter;
}
 

@Override
public Iterable<U> buildUpdates(JavaPairRDD<K, M> newData) {
  return JavaConversions.asJavaIterable(scalaManager.buildUpdates(newData.rdd()));
}
 

public static VSensorConfig vsensor(VsConf vs){
 VSensorConfig v=new VSensorConfig();
 v.setMainClass(vs.processing().className());
 v.setDescription(vs.description());
 v.setName(vs.name());
 v.setIsTimeStampUnique(vs.processing().uniqueTimestamp());
 if (vs.poolSize().isDefined())
   v.setLifeCyclePoolSize(((Integer)vs.poolSize().get()));
 if (vs.processing().rate().isDefined())
   v.setOutputStreamRate(((Integer)vs.processing().rate().get()));
 v.setPriority(vs.priority());
 KeyValueImp [] addr=new KeyValueImp[vs.address().size()];
    Iterable<String> keys=JavaConversions.asJavaIterable(vs.address().keys());
    int i=0;
 for (String k:keys){
  addr[i]=new KeyValueImp(k,vs.address().apply(k));
  i++;
 }
 v.setAddressing(addr);
 InputStream[] is=new InputStream[vs.streams().size()];
 for (int j=0;j<is.length;j++){
  is[j]=stream(vs.streams().apply(j));
 }
 v.setInputStreams(is);
 if (vs.processing().webInput().isDefined()){
  WebInputConf wic=vs.processing().webInput().get();
  v.setWebParameterPassword(wic.password());
  WebInput[] wi=new WebInput[wic.commands().size()];
  for (int j=0;j<wi.length;j++){
	  wi[j]=webInput(wic.commands().apply(j));
  }
  v.setWebInput(wi);
 }
 DataField [] out=new DataField[(vs.processing().output().size())];
 for (int j=0;j<out.length;j++){
  out[j]=dataField(vs.processing().output().apply(j));
 }
 v.setOutputStructure(out);
 Map<String,String> init=vs.processing().initParams();
 ArrayList<KeyValue> ini=new ArrayList<KeyValue>();
    Iterable<String> initkeys=JavaConversions.asJavaIterable(init.keys());
 for (String ik:initkeys){
  logger.trace("keys:"+ik);
  ini.add(new KeyValueImp(ik.toLowerCase(),init.apply(ik)));
 }
 v.setMainClassInitialParams(ini);
 
 StorageConfig st=new StorageConfig();
 if (vs.storageSize().isDefined())
  st.setStorageSize(vs.storageSize().get());
 if (vs.storage().isDefined()){
StorageConf sc=vs.storage().get();
if (sc.identifier().isDefined())
  st.setIdentifier(sc.identifier().get());
st.setJdbcDriver(sc.driver());
st.setJdbcURL(sc.url());
st.setJdbcUsername(sc.user());
st.setJdbcPassword(sc.pass());		
 }
 if (st.getStorageSize()!=null || st.getJdbcURL()!=null)
v.setStorage(st);
 return v;
}