Java Code Examples for org.apache.flink.configuration.Configuration#setBytes()

The following examples show how to use org.apache.flink.configuration.Configuration#setBytes() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: toolbox   File: DistributedVI.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(){

        Attribute seq_id = this.dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return this.dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 2
Source Project: toolbox   File: DistributedVI.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(List<Variable> latentVariables){

        Attribute seq_id = this.dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return this.dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 3
Source Project: toolbox   File: dVMP.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize,batchConverter)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 4
Source Project: toolbox   File: dVMPv1.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 5
Source Project: toolbox   File: ParallelVB.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosteriorAssignment> computePosteriorAssignment(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInferenceAssignment())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 6
Source Project: toolbox   File: ParallelVB.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink, List<Variable> latentVariables){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(LATENT_VARS, Serialization.serializeObject(latentVariables));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 7
Source Project: toolbox   File: ParallelVB.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 8
Source Project: toolbox   File: dVMPv1.java    License: Apache License 2.0 6 votes vote down vote up
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink){

        Attribute seq_id = dataFlink.getAttributes().getSeq_id();
        if (seq_id==null)
            throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute");

        try{
            Configuration config = new Configuration();
            config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
            config.setBytes(SVB, Serialization.serializeObject(svb));

            return dataFlink
                    .getBatchedDataSet(this.batchSize)
                    .flatMap(new ParallelVBMapInference())
                    .withParameters(config);

        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }

    }
 
Example 9
Source Project: Flink-CEPplus   File: DistributedCache.java    License: Apache License 2.0 5 votes vote down vote up
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
 
Example 10
Source Project: flink   File: DistributedCache.java    License: Apache License 2.0 5 votes vote down vote up
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
 
Example 11
public static void storeSerializedInstance(
    Configuration configuration, ConfigOption<byte[]> option, Object instance) {
  try {
    byte[] bytes = InstantiationUtil.serializeObject(instance);
    configuration.setBytes(option.key(), bytes);
  } catch (IOException e) {
    throw new IllegalStateException(e);
  }
}
 
Example 12
Source Project: flink   File: DistributedCache.java    License: Apache License 2.0 5 votes vote down vote up
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
 
Example 13
Source Project: toolbox   File: ConversionToBatches.java    License: Apache License 2.0 5 votes vote down vote up
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatches(DataFlink<T> data, int batchSize){

        try{
            Configuration config = new Configuration();
            config.setInteger(BATCH_SIZE, batchSize);
            config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes()));

            return data.getDataSet().mapPartition(new DataBatch<T>()).withParameters(config);
        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }
    }
 
Example 14
Source Project: toolbox   File: ConversionToBatches.java    License: Apache License 2.0 5 votes vote down vote up
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatchesBySeqID(DataFlink<T> data, int batchSize){

        try{
            Configuration config = new Configuration();
            config.setInteger(BATCH_SIZE, batchSize);
            config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes()));

            return data.getDataSet().mapPartition(new DataBatchBySeqID<T>()).withParameters(config);
        }catch(Exception ex){
            throw new UndeclaredThrowableException(ex);
        }
    }
 
Example 15
/**
 * {@inheritDoc}
 */
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate) {

    try {

        this.initLearning();

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
        config.setBytes(EFBN_NAME, Serialization.serializeObject(efBayesianNetwork));

        DataSet<DataInstance> dataset = dataUpdate.getDataSet();

        this.sumSS = dataset.mapPartition(new SufficientSatisticsMAP())
                .withParameters(config)
                .reduce(new SufficientSatisticsReduce())
                .collect().get(0);

        //Add the prior
        sumSS.sum(efBayesianNetwork.createInitSufficientStatistics());

        JobExecutionResult result = dataset.getExecutionEnvironment().getLastJobExecutionResult();

        numInstances = result.getAccumulatorResult(ParallelMaximumLikelihood2.COUNTER_NAME+"_"+this.dag.getName());
        numInstances++;//Initial counts

    }catch(Exception ex){
        throw new UndeclaredThrowableException(ex);
    }

    return this.getLogMarginalProbability();
}
 
Example 16
Source Project: toolbox   File: StochasticVI.java    License: Apache License 2.0 5 votes vote down vote up
public static double computeELBO(DataFlink<DataInstance> dataFlink, SVB svb, Function2<DataFlink<DataInstance>,Integer,DataSet<DataOnMemory<DataInstance>>> batchConverter){

        svb.setOutput(false);
        double elbo =  svb.getPlateuStructure().getNonReplictedNodes().mapToDouble(node -> svb.getPlateuStructure().getVMP().computeELBO(node)).sum();

        try {

            Configuration config = new Configuration();
            config.setBytes(SVB, Serialization.serializeObject(svb));
            config.setBytes(PRIOR, Serialization.serializeObject(svb.getPlateuStructure().getPlateauNaturalParameterPosterior()));

            DataSet<DataOnMemory<DataInstance>> batches;
            if (batchConverter!=null)
                batches= dataFlink.getBatchedDataSet(svb.getWindowsSize(),batchConverter);
            else
                batches= dataFlink.getBatchedDataSet(svb.getWindowsSize());

            elbo += batches.map(new ParallelVBMapELBO())
                    .withParameters(config)
                    .reduce(new ReduceFunction<Double>() {
                        @Override
                        public Double reduce(Double aDouble, Double t1) throws Exception {
                            return aDouble + t1;
                        }
                    }).collect().get(0);
        } catch (Exception e) {
            e.printStackTrace();
        }

        svb.setOutput(true);

        return elbo;
    }
 
Example 17
/**
 * {@inheritDoc}
 */
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate) {
    try {
        Configuration config = new Configuration();
        config.setString(BN_NAME, this.dag.getName());
        config.setBytes(EFBN_NAME, Serialization.serializeObject(efBayesianNetwork));

        DataSet<DataInstance> dataset = dataUpdate.getDataSet();
        this.sumSS = dataset.map(new SufficientSatisticsMAP())
                .withParameters(config)
                .reduce(new SufficientSatisticsReduce())
                .collect().get(0);

        //Add the prior
        sumSS.sum(efBayesianNetwork.createInitSufficientStatistics());

        JobExecutionResult result = dataset.getExecutionEnvironment().getLastJobExecutionResult();

        numInstances = result.getAccumulatorResult(ParallelMaximumLikelihood.COUNTER_NAME+"_"+this.dag.getName());
        numInstances++;//Initial counts

    }catch(Exception ex){
        throw new UndeclaredThrowableException(ex);
    }

    return this.getLogMarginalProbability();
}
 
Example 18
Source Project: Flink-CEPplus   File: InstantiationUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException {
	byte[] bytes = serializeObject(o);
	config.setBytes(key, bytes);
}
 
Example 19
Source Project: flink   File: InstantiationUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException {
	byte[] bytes = serializeObject(o);
	config.setBytes(key, bytes);
}
 
Example 20
Source Project: toolbox   File: DistributedVI.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public double updateModel(DataFlink<DataInstance> dataUpdate){

    try{
        final ExecutionEnvironment env = dataUpdate.getDataSet().getExecutionEnvironment();

        // get input data
        CompoundVector parameterPrior = this.svb.getNaturalParameterPrior();

        DataSet<CompoundVector> paramSet = env.fromElements(parameterPrior);

        ConvergenceCriterion convergenceELBO;
        if(timeLimit == -1) {
            convergenceELBO = new ConvergenceELBO(this.globalThreshold, System.nanoTime());
        }
        else {
            convergenceELBO = new ConvergenceELBObyTime(this.timeLimit, System.nanoTime());
            this.setMaximumGlobalIterations(5000);
        }
        // set number of bulk iterations for KMeans algorithm
        IterativeDataSet<CompoundVector> loop = paramSet.iterate(maximumGlobalIterations)
                .registerAggregationConvergenceCriterion("ELBO_" + this.dag.getName(), new DoubleSumAggregator(),convergenceELBO);

        Configuration config = new Configuration();
        config.setString(ParameterLearningAlgorithm.BN_NAME, this.dag.getName());
        config.setBytes(SVB, Serialization.serializeObject(svb));

        //We add an empty batched data set to emit the updated prior.
        DataOnMemory<DataInstance> emtpyBatch = new DataOnMemoryListContainer<DataInstance>(dataUpdate.getAttributes());
        DataSet<DataOnMemory<DataInstance>> unionData = null;

        unionData =
                dataUpdate.getBatchedDataSet(this.batchSize)
                        .union(env.fromCollection(Arrays.asList(emtpyBatch),
                                TypeExtractor.getForClass((Class<DataOnMemory<DataInstance>>) Class.forName("eu.amidst.core.datastream.DataOnMemory"))));

        DataSet<CompoundVector> newparamSet =
                unionData
                .map(new ParallelVBMap(randomStart, idenitifableModelling))
                .withParameters(config)
                .withBroadcastSet(loop, "VB_PARAMS_" + this.dag.getName())
                .reduce(new ParallelVBReduce());

        // feed new centroids back into next iteration
        DataSet<CompoundVector> finlparamSet = loop.closeWith(newparamSet);

        parameterPrior = finlparamSet.collect().get(0);

        this.svb.updateNaturalParameterPosteriors(parameterPrior);

        this.svb.updateNaturalParameterPrior(parameterPrior);

        if(timeLimit == -1)
            this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO();
        else
            this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();

        this.svb.applyTransition();

    }catch(Exception ex){
        throw new RuntimeException(ex.getMessage());
    }

    this.randomStart=false;

    return this.getLogMarginalProbability();
}