Java Code Examples for org.apache.flink.configuration.Configuration#setInteger()

The following examples show how to use org.apache.flink.configuration.Configuration#setInteger() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TaskManagerServicesConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
}
 
Example 2
Source File: ExecutionGraphDeploymentWithBlobServerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setupBlobServer() throws IOException {
	Configuration config = new Configuration();
	// always offload the serialized job and task information
	config.setInteger(BlobServerOptions.OFFLOAD_MINSIZE, 0);
	blobServer = Mockito.spy(new BlobServer(config, new VoidBlobStore()));
	blobWriter = blobServer;
	blobCache = blobServer;

	seenHashes.clear();

	// verify that we do not upload the same content more than once
	doAnswer(
		invocation -> {
			PermanentBlobKey key = (PermanentBlobKey) invocation.callRealMethod();

			assertTrue(seenHashes.add(key.getHash()));

			return key;
		}
	).when(blobServer).putPermanent(any(JobID.class), Matchers.<byte[]>any());

	blobServer.start();
}
 
Example 3
Source File: YARNHighAvailabilityITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);

	flinkConfiguration.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	final int minMemory = 100;
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, minMemory);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 
Example 4
Source File: HAQueryableStateRocksDBBackendITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example 5
Source File: PythonTableFunctionOperatorTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testFinishBundleTriggeredByCount() throws Exception {
	Configuration conf = new Configuration();
	conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 2);
	OneInputStreamOperatorTestHarness<IN, OUT> testHarness = getTestHarness(conf, JoinRelType.INNER);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1));
	assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 1L), initialTime + 2));
	expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 0L)));
	expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 1L, 1L)));

	assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 6
Source File: DataFlinkWriterTest.java    From toolbox with Apache License 2.0 5 votes vote down vote up
public static void test1() throws Exception {
    //Set-up Flink session.
    Configuration conf = new Configuration();
    conf.setInteger("taskmanager.network.numberOfBuffers", 12000);
    final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf);
            env.getConfig().disableSysoutLogging();         env.setParallelism(Main.PARALLELISM);

    DataFlink<DataInstance> dataFlink = DataFlinkLoader.loadDataFromFile(env,
            "../datasets/simulated/test_not_modify/SmallDataSet.arff", false);

    DataFlinkWriter.writeDataToARFFFolder(dataFlink, "../datasets/simulated/tmp_2.arff");

}
 
Example 7
Source File: DataFlinkLoaderTest.java    From toolbox with Apache License 2.0 5 votes vote down vote up
public static void test2() throws Exception {
    //Set-up Flink session.
    Configuration conf = new Configuration();
    conf.setInteger("taskmanager.network.numberOfBuffers", 12000);
    final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf);
            env.getConfig().disableSysoutLogging();         env.setParallelism(Main.PARALLELISM);

    DataFlink<DataInstance> dataFlink = DataFlinkLoader.loadDataFromFile(env,
            "../datasets/simulated/test_not_modify/SmallDataSet.arff", false);
    DataSet<DataOnMemory<DataInstance>> data = dataFlink.getBatchedDataSet(3);

    data.print();

    List<DataOnMemory<DataInstance>> batchList = data.collect();

    int size = 0;
    for (DataOnMemory<DataInstance> dataInstanceDataBatch : batchList) {
        if (Main.VERBOSE) System.out.println("Batch :" + dataInstanceDataBatch.getList().size());
        size += dataInstanceDataBatch.getList().size();
    }
    assertEquals(16, size);

    List<DataInstance> instanceList = batchList.stream().flatMap(batch -> batch.getList().stream()).collect(Collectors.toList());

    assertEquals(16, instanceList.size());
    List<String> names = Arrays.asList("A", "B", "C", "D", "E", "G");
    List<Integer> states = Arrays.asList(2, 3, 2, 2, 2, -1);

    List<Attribute> atts = dataFlink.getAttributes().getListOfNonSpecialAttributes();
    for (int i = 0; i < names.size(); i++) {
        if (Main.VERBOSE) System.out.println(names.get(i));
        assertEquals(atts.get(i).getName(), names.get(i));
        assertEquals(atts.get(i).getNumberOfStates(), states.get(i).intValue());
    }
}
 
Example 8
Source File: BackPressureITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration createBackPressureSamplingConfiguration() {
	final Configuration configuration = new Configuration();
	configuration.setInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL, 1000);
	configuration.setInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES, 1);
	configuration.setInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL, Integer.MAX_VALUE);
	return configuration;
}
 
Example 9
Source File: RemoteExecutor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public RemoteExecutor(InetSocketAddress inet, Configuration clientConfiguration,
		List<URL> jarFiles, List<URL> globalClasspaths) {
	this.clientConfiguration = clientConfiguration;
	this.jarFiles = jarFiles;
	this.globalClasspaths = globalClasspaths;

	clientConfiguration.setString(JobManagerOptions.ADDRESS, inet.getHostName());
	clientConfiguration.setInteger(JobManagerOptions.PORT, inet.getPort());
	clientConfiguration.setInteger(RestOptions.PORT, inet.getPort());
}
 
Example 10
Source File: DefaultCLITest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the configuration is properly passed via the DefaultCLI to the
 * created ClusterDescriptor.
 */
@Test
public void testConfigurationPassing() throws Exception {
	final Configuration configuration = getConfiguration();

	final String localhost = "localhost";
	final int port = 1234;

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String[] args = {};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(localhost));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(port));
}
 
Example 11
Source File: SimpleRecoveryFixedDelayRestartStrategyITBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	config.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	config.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "100 ms");

	return config;
}
 
Example 12
Source File: CancelingTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, "4096");
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}
 
Example 13
Source File: RestartStrategyFactoryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void createRestartStrategyFactory_noRestartStrategyButAttemptsConfigured_returnsNoOrFixedIfCheckpointingEnabledRestartStrategyFactory() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);

	final RestartStrategyFactory restartStrategyFactory = RestartStrategyFactory.createRestartStrategyFactory(configuration);

	assertThat(restartStrategyFactory, instanceOf(NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.class));
}
 
Example 14
Source File: RestServerEndpointITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getBaseConfig() {
	final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress();

	final Configuration config = new Configuration();
	config.setString(RestOptions.BIND_PORT, "0");
	config.setString(RestOptions.BIND_ADDRESS, loopbackAddress);
	config.setString(RestOptions.ADDRESS, loopbackAddress);
	config.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH);
	config.setInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH);
	return config;
}
 
Example 15
Source File: ZooKeeperTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Sets all necessary configuration keys to operate in {@link HighAvailabilityMode#ZOOKEEPER}.
 *
 * @param config            Configuration to use
 * @param zooKeeperQuorum   ZooKeeper quorum to connect to
 * @param fsStateHandlePath Base path for file system state backend (for checkpoints and
 *                          recovery)
 * @return The modified configuration to operate in {@link HighAvailabilityMode#ZOOKEEPER}.
 */
public static Configuration configureZooKeeperHA(
		Configuration config,
		String zooKeeperQuorum,
		String fsStateHandlePath) {

	checkNotNull(config, "Configuration");
	checkNotNull(zooKeeperQuorum, "ZooKeeper quorum");
	checkNotNull(fsStateHandlePath, "File state handle backend path");

	// ZooKeeper recovery mode
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperQuorum);

	int connTimeout = 5000;
	if (System.getenv().containsKey("CI")) {
		// The regular timeout is to aggressive for Travis and connections are often lost.
		connTimeout = 30000;
	}

	config.setInteger(HighAvailabilityOptions.ZOOKEEPER_CONNECTION_TIMEOUT, connTimeout);
	config.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, connTimeout);

	// File system state backend
	config.setString(CheckpointingOptions.STATE_BACKEND, "FILESYSTEM");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, fsStateHandlePath + "/checkpoints");
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, fsStateHandlePath + "/recovery");

	// Akka failure detection and execution retries
	config.setString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL, "1000 ms");
	config.setString(AkkaOptions.WATCH_HEARTBEAT_PAUSE, "6 s");
	config.setInteger(AkkaOptions.WATCH_THRESHOLD, 9);
	config.setString(AkkaOptions.ASK_TIMEOUT, "100 s");
	config.setString(HighAvailabilityOptions.HA_JOB_DELAY, "10 s");

	return config;
}
 
Example 16
Source File: ExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 
Example 17
Source File: LimitedConnectionsConfigurationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * This test validates that the File System is correctly wrapped by the
 * file system factories when the corresponding entries are in the configuration.
 */
@Test
public void testConfiguration() throws Exception {
	final String fsScheme = TestFileSystem.SCHEME;

	// nothing configured, we should get a regular file system
	FileSystem schemeFs = FileSystem.get(URI.create(fsScheme + ":///a/b/c"));
	FileSystem localFs = FileSystem.get(tempDir.newFile().toURI());

	assertFalse(schemeFs instanceof LimitedConnectionsFileSystem);
	assertFalse(localFs instanceof LimitedConnectionsFileSystem);

	// configure some limits, which should cause "fsScheme" to be limited

	final Configuration config = new Configuration();
	config.setInteger("fs." + fsScheme + ".limit.total", 42);
	config.setInteger("fs." + fsScheme + ".limit.input", 11);
	config.setInteger("fs." + fsScheme + ".limit.output", 40);
	config.setInteger("fs." + fsScheme + ".limit.timeout", 12345);
	config.setInteger("fs." + fsScheme + ".limit.stream-timeout", 98765);

	try {
		FileSystem.initialize(config);

		schemeFs = FileSystem.get(URI.create(fsScheme + ":///a/b/c"));
		localFs = FileSystem.get(tempDir.newFile().toURI());

		assertTrue(schemeFs instanceof LimitedConnectionsFileSystem);
		assertFalse(localFs instanceof LimitedConnectionsFileSystem);

		LimitedConnectionsFileSystem limitedFs = (LimitedConnectionsFileSystem) schemeFs;
		assertEquals(42, limitedFs.getMaxNumOpenStreamsTotal());
		assertEquals(11, limitedFs.getMaxNumOpenInputStreams());
		assertEquals(40, limitedFs.getMaxNumOpenOutputStreams());
		assertEquals(12345, limitedFs.getStreamOpenTimeout());
		assertEquals(98765, limitedFs.getStreamInactivityTimeout());
	}
	finally {
		// clear all settings
		FileSystem.initialize(new Configuration());
	}
}
 
Example 18
Source File: StochasticVIExample.java    From toolbox with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

        boolean hadoop_cluster = false;

        if (args.length>1){
            hadoop_cluster = Boolean.parseBoolean(args[0]);
        }

        final ExecutionEnvironment env;

        //Set-up Flink session.
        if(hadoop_cluster){
            env = ExecutionEnvironment.getExecutionEnvironment();
            env.getConfig().disableSysoutLogging();
        }else{
            Configuration conf = new Configuration();
            conf.setInteger("taskmanager.network.numberOfBuffers", 12000);
            conf.setInteger("taskmanager.numberOfTaskSlots",Main.PARALLELISM);
            env = ExecutionEnvironment.createLocalEnvironment(conf);
            env.setParallelism(Main.PARALLELISM);
            env.getConfig().disableSysoutLogging();
        }
        //generate a random dataset
        DataFlink<DataInstance> dataFlink = new DataSetGenerator().generate(env,1234,1000,5,0);

        //Creates a DAG with the NaiveBayes structure for the random dataset
        DAG dag = DAGGenerator.getNaiveBayesStructure(dataFlink.getAttributes(), "DiscreteVar4");
        System.out.println(dag.toString());


        //Create the  Learner object
        ParameterLearningAlgorithm learningAlgorithmFlink =
                new StochasticVI();

        //Learning parameters
        learningAlgorithmFlink.setBatchSize(10);
        learningAlgorithmFlink.setDAG(dag);

        //Initialize the learning process
        learningAlgorithmFlink.initLearning();

        //Learn from the flink data
        learningAlgorithmFlink.updateModel(dataFlink);

        //Specific parameters for the algorithm
        ((StochasticVI)learningAlgorithmFlink).setLearningFactor(0.7);
        ((StochasticVI)learningAlgorithmFlink).setDataSetSize((int) dataFlink.getDataSet().count());


        //Print the learnt BN
        BayesianNetwork bn = learningAlgorithmFlink.getLearntBayesianNetwork();
        System.out.println(bn);



    }
 
Example 19
Source File: ParallelVBTest.java    From toolbox with Apache License 2.0 4 votes vote down vote up
public void testingMLParallelAsiaHidden() throws IOException, ClassNotFoundException {

        //Set-up Flink session.
        Configuration conf = new Configuration();
        conf.setInteger("taskmanager.network.numberOfBuffers", 12000);
        final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf);
                env.getConfig().disableSysoutLogging();         env.setParallelism(Main.PARALLELISM);

        // load the true Asia Bayesian network
        BayesianNetwork asianet = BayesianNetworkLoader.loadFromFile("../networks/dataWeka/asia.bn");
        asianet.randomInitialization(new Random(0));
        if (Main.VERBOSE) System.out.println("\nAsia network \n ");
        //if (Main.VERBOSE) System.out.println(asianet.getDAG().outputString());
        if (Main.VERBOSE) System.out.println(asianet.toString());

        //Sampling from Asia BN
        BayesianNetworkSampler sampler = new BayesianNetworkSampler(asianet);
        sampler.setSeed(0);
        //Load the sampled data
        DataStream<DataInstance> data = sampler.sampleToDataStream(10000);
        sampler.setHiddenVar(asianet.getVariables().getVariableById(7));
        DataStreamWriter.writeDataToFile(data, "../datasets/simulated/tmp.arff");

        DataFlink<DataInstance> dataFlink = DataFlinkLoader.loadDataFromFile(env, "../datasets/simulated/tmp.arff", false);

        //Structure learning is excluded from the test, i.e., we use directly the initial Asia network structure
        // and just learn then test the parameter learning

        long start = System.nanoTime();

        //Parameter Learning
        ParallelVB parallelVB = new ParallelVB();
        parallelVB.setOutput(true);
        parallelVB.setSeed(5);
        parallelVB.setBatchSize(100);
        parallelVB.setLocalThreshold(0.001);
        parallelVB.setGlobalThreshold(0.05);
        parallelVB.setMaximumLocalIterations(100);
        parallelVB.setMaximumGlobalIterations(100);


        parallelVB.setDAG(asianet.getDAG());
        parallelVB.initLearning();
        parallelVB.updateModel(dataFlink);
        BayesianNetwork bnet = parallelVB.getLearntBayesianNetwork();

        if (Main.VERBOSE) System.out.println(bnet.toString());

        long duration = (System.nanoTime() - start) / 1;
        double seconds = duration / 1000000000.0;
        if (Main.VERBOSE) System.out.println("Running time: \n" + seconds + " secs");

    }
 
Example 20
Source File: DynamicParallelVBTest.java    From toolbox with Apache License 2.0 4 votes vote down vote up
public static void testUpdateN(String networkName, double threshold) throws Exception {
    //Set-up Flink session.
    Configuration conf = new Configuration();
    conf.setInteger("taskmanager.network.numberOfBuffers", 12000);
    final ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(conf);
            env.getConfig().disableSysoutLogging();         env.setParallelism(Main.PARALLELISM);

    DynamicBayesianNetwork dbn = DynamicBayesianNetworkLoader.loadFromFile("../networks/simulated/" + networkName+".dbn");
    dbn.randomInitialization(new Random(0));

    if (Main.VERBOSE) System.out.println(dbn.toString());


    DataFlink<DynamicDataInstance> data0 = DataFlinkLoader.loadDynamicDataFromFolder(env,
            "../datasets/simulated/cajaMarSynthetic/data0.arff", false);
    dbn.getDynamicVariables().setAttributes(data0.getAttributes());

    DynamicParallelVB learn = new DynamicParallelVB();
    learn.setMaximumGlobalIterations(20);
    learn.setGlobalThreshold(0.0001);
    learn.setLocalThreshold(0.0001);
    learn.setMaximumLocalIterations(200);
    learn.setBatchSize(BATCHSIZE);
    learn.setDAG(dbn.getDynamicDAG());
    learn.setOutput(true);
    learn.initLearning();

    if (Main.VERBOSE) System.out.println("--------------- DATA " + 0 + " --------------------------");
    learn.updateModelWithNewTimeSlice(0, data0);

    for (int i = 1; i < NSETS; i++) {
        if (Main.VERBOSE) System.out.println("--------------- DATA " + i + " --------------------------");
        DataFlink<DynamicDataInstance> dataNew = DataFlinkLoader.loadDynamicDataFromFolder(env,
                "../datasets/simulated/cajaMarSynthetic/data" + i + ".arff", false);
        learn.updateModelWithNewTimeSlice(i, dataNew);
        assertEquals(SAMPLESIZE, learn.dataPosteriorDataSet.count());
    }

    if (Main.VERBOSE) System.out.println(learn.getLearntDynamicBayesianNetwork());
    if (threshold>0) assertEquals(true, dbn.equalDBNs(learn.getLearntDynamicBayesianNetwork(), threshold));
    //learn.getLearntBayesianNetwork()
}