Java Code Examples for org.apache.flink.configuration.Configuration

The following examples show how to use org.apache.flink.configuration.Configuration. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: flink   Source File: PythonProgramOptionsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateProgramOptionsWithLongOptions() throws CliArgsException {
	String[] args = {
		"--python", "xxx.py",
		"--pyModule", "xxx",
		"--pyFiles", "/absolute/a.py,relative/b.py,relative/c.py",
		"--pyRequirements", "d.txt#e_dir",
		"--pyExecutable", "/usr/bin/python",
		"--pyArchives", "g.zip,h.zip#data,h.zip#data2",
		"userarg1", "userarg2"
	};

	CommandLine line = CliFrontendParser.parse(options, args, false);
	PythonProgramOptions programOptions = (PythonProgramOptions) ProgramOptions.create(line);
	Configuration config = new Configuration();
	programOptions.applyToConfiguration(config);
	assertEquals("/absolute/a.py,relative/b.py,relative/c.py", config.get(PythonOptions.PYTHON_FILES));
	assertEquals("d.txt#e_dir", config.get(PYTHON_REQUIREMENTS));
	assertEquals("g.zip,h.zip#data,h.zip#data2", config.get(PythonOptions.PYTHON_ARCHIVES));
	assertEquals("/usr/bin/python", config.get(PYTHON_EXECUTABLE));
	assertArrayEquals(
		new String[] {"--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2"},
		programOptions.getProgramArgs());
}
 
Example 2
Source Project: Flink-CEPplus   Source File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 3
Source Project: flink   Source File: ClientTest.java    License: Apache License 2.0 6 votes vote down vote up
private void launchMultiExecuteJob(final boolean enforceSingleJobExecution) throws ProgramInvocationException {
	try (final ClusterClient<?> clusterClient =
				new MiniClusterClient(new Configuration(), MINI_CLUSTER_RESOURCE.getMiniCluster())) {

		final PackagedProgram program = PackagedProgram.newBuilder()
				.setEntryPointClassName(TestMultiExecute.class.getName())
				.build();

		final Configuration configuration = fromPackagedProgram(program, 1, false);

		ClientUtils.executeProgram(
				new TestExecutorServiceLoader(clusterClient, plan),
				configuration,
				program,
				enforceSingleJobExecution,
				false);
	}
}
 
Example 4
Source Project: alibaba-flink-connectors   Source File: SlsRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
public SlsRecordReader(
		String endPoint,
		Configuration properties,
		String project,
		String logStore,
		int startInSec,
		int stopInSec,
		int maxRetryTime,
		int batchGetSize,
		List<Shard> initShardList,
		String consumerGroup) {
	this.endPoint = endPoint;
	this.project = project;
	this.logStore = logStore;
	this.startInSec = startInSec;
	this.stopInSec = stopInSec;
	this.maxRetryTime = maxRetryTime;
	this.batchGetSize = batchGetSize;
	setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
	this.properties = properties;
	this.consumerGroup = consumerGroup;
}
 
Example 5
Source Project: flink   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 6
Source Project: flink   Source File: TaskExecutorSubmissionTest.java    License: Apache License 2.0 6 votes vote down vote up
private TaskDeploymentDescriptor createTestTaskDeploymentDescriptor(
	String taskName,
	ExecutionAttemptID eid,
	Class<? extends AbstractInvokable> abstractInvokable,
	int maxNumberOfSubtasks,
	Collection<ResultPartitionDeploymentDescriptor> producedPartitions,
	Collection<InputGateDeploymentDescriptor> inputGates
) throws IOException {
	Preconditions.checkNotNull(producedPartitions);
	Preconditions.checkNotNull(inputGates);
	return createTaskDeploymentDescriptor(
		jobId, testName.getMethodName(), eid,
		new SerializedValue<>(new ExecutionConfig()), taskName, maxNumberOfSubtasks, 0, 1, 0,
		new Configuration(), new Configuration(), abstractInvokable.getName(),
		producedPartitions,
		inputGates,
		Collections.emptyList(),
		Collections.emptyList(),
		0);
}
 
Example 7
Source Project: flink   Source File: TaskManagerRunnerConfigurationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDefaultFsParameterLoading() throws Exception {
	try {
		final File tmpDir = temporaryFolder.newFolder();
		final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);

		final URI defaultFS = new URI("otherFS", null, "localhost", 1234, null, null, null);

		final PrintWriter pw1 = new PrintWriter(confFile);
		pw1.println("fs.default-scheme: " + defaultFS);
		pw1.close();

		String[] args = new String[] {"--configDir", tmpDir.toString()};
		Configuration configuration = TaskManagerRunner.loadConfiguration(args);
		FileSystem.initialize(configuration);

		assertEquals(defaultFS, FileSystem.getDefaultFsUri());
	}
	finally {
		// reset FS settings
		FileSystem.initialize(new Configuration());
	}
}
 
Example 8
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example 9
Source Project: Alink   Source File: SortUtilsNext.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	this.taskId = getRuntimeContext().getIndexOfThisSubtask();

	LOG.info("{} open.", getRuntimeContext().getTaskName());

	List<Tuple2<Integer, Long>> allCnt = getRuntimeContext().getBroadcastVariable("cnt");

	for (Tuple2<Integer, Long> localCnt : allCnt) {
		if (localCnt.f0 == taskId) {
			cnt = localCnt.f1.intValue();
			break;
		}
	}
}
 
Example 10
private static void initializeActorSystem(String hostname) throws UnknownHostException {
  synchronized (actorSystemLock) {
    if (actorSystem == null) {
      Configuration config = new Configuration();
      Option<scala.Tuple2<String, Object>> remoting = new Some<>(new scala.Tuple2<String, Object>(hostname, 0));

      Config akkaConfig = AkkaUtils.getAkkaConfig(config, remoting);

      LOG.info("Start actory system.");
      actorSystem = ActorSystem.create("queryableWindow", akkaConfig);
      actorSystemUsers = 1;
    } else {
      LOG.info("Actor system has already been started.");
      actorSystemUsers++;
    }
  }
}
 
Example 11
Source Project: flink   Source File: StreamingCustomInputSplitProgram.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
			Configuration config = new Configuration();

	config.setString(AkkaOptions.ASK_TIMEOUT, "5 s");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> data = env.createInput(new CustomInputFormat());

	data.map(new MapFunction<Integer, Tuple2<Integer, Double>>() {
		@Override
		public Tuple2<Integer, Double> map(Integer value) throws Exception {
			return new Tuple2<Integer, Double>(value, value * 0.5);
		}
	}).addSink(new DiscardingSink<>());

	env.execute();
}
 
Example 12
Source Project: Flink-CEPplus   Source File: BlobServerSSLTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFailedToInitWithInvalidSslKeystoreConfigured() {
	final Configuration config = new Configuration();

	config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true);
	config.setString(SecurityOptions.SSL_KEYSTORE, "invalid.keystore");
	config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_TRUSTSTORE, "invalid.keystore");
	config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password");

	try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) {
		fail();
	} catch (Exception e) {
		findThrowable(e, IOException.class);
		findThrowableWithMessage(e, "Failed to initialize SSL for the blob server");
	}
}
 
Example 13
Source Project: flink   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 14
Source Project: flink   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: RestClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 
Example 16
Source Project: kylin   Source File: FlinkCubingByLayer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 
Example 17
Source Project: flink   Source File: LeaderElectionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: IPv6HostnamesITCase.java    License: Apache License 2.0 5 votes vote down vote up
private Configuration getConfiguration() {
	final Inet6Address ipv6address = getLocalIPv6Address();
	if (ipv6address == null) {
		throw new AssumptionViolatedException("--- Cannot find a non-loopback local IPv6 address that Akka/Netty can bind to; skipping IPv6HostnamesITCase");
	}
	final String addressString = ipv6address.getHostAddress();
	log.info("Test will use IPv6 address " + addressString + " for connection tests");

	Configuration config = new Configuration();
	config.setString(JobManagerOptions.ADDRESS, addressString);
	config.setString(TaskManagerOptions.HOST, addressString);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m");
	return config;
}
 
Example 19
Source Project: flink   Source File: TestBaseUtils.java    License: Apache License 2.0 5 votes vote down vote up
protected static Collection<Object[]> toParameterList(Configuration ... testConfigs) {
	ArrayList<Object[]> configs = new ArrayList<>();
	for (Configuration testConfig : testConfigs) {
		Object[] c = { testConfig };
		configs.add(c);
	}
	return configs;
}
 
Example 20
Source Project: flink   Source File: FlinkKinesisConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testFetcherShouldNotBeRestoringFromFailureIfNotRestoringFromCheckpoint() throws Exception {
	KinesisDataFetcher mockedFetcher = mockKinesisDataFetcher();

	// assume the given config is correct
	PowerMockito.mockStatic(KinesisConfigUtil.class);
	PowerMockito.doNothing().when(KinesisConfigUtil.class);

	TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
		"fakeStream", new Properties(), 10, 2);
	consumer.open(new Configuration());
	consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
}
 
Example 21
Source Project: flink   Source File: RetryingRegistrationConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
public static RetryingRegistrationConfiguration fromConfiguration(final Configuration configuration) {
	long initialRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT);
	long maxRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT);
	long errorDelayMillis = configuration.getLong(ClusterOptions.ERROR_REGISTRATION_DELAY);
	long refusedDelayMillis = configuration.getLong(ClusterOptions.REFUSED_REGISTRATION_DELAY);

	return new RetryingRegistrationConfiguration(
		initialRegistrationTimeoutMillis,
		maxRegistrationTimeoutMillis,
		errorDelayMillis,
		refusedDelayMillis);
}
 
Example 22
Source Project: flink   Source File: TestProcessBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public TestProcessBuilder addConfigAsMainClassArgs(Configuration config) {
	for (Entry<String, String> keyValue: config.toMap().entrySet()) {
		addMainClassArg("--" + keyValue.getKey());
		addMainClassArg(keyValue.getValue());
	}
	return this;
}
 
Example 23
Source Project: flink   Source File: MesosTaskManagerParametersTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUriParametersDefault() throws Exception {
	Configuration config = new Configuration();

	MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
	assertEquals(params.uris().size(), 0);
}
 
Example 24
Source Project: flink   Source File: StreamFlatMapTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	if (closeCalled) {
		Assert.fail("Close called before open.");
	}
	openCalled = true;
}
 
Example 25
Source Project: Flink-CEPplus   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void readWithParseQuotedStrings() {
	try {
		final String fileContent = "\"ab\\\"c\"|\"def\"\n\"ghijk\"|\"abc\"";
		final FileInputSplit split = createTempFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class);
		format.enableQuotedStringParsing('"');

		format.configure(parameters);
		format.open(split);

		Value[] values = new Value[] { new StringValue(), new StringValue()};

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ab\\\"c", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ghijk", ((StringValue) values[0]).getValue());
		assertEquals("abc", ((StringValue) values[1]).getValue());

	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 26
Source Project: flink   Source File: SecurityUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSecurityContext() throws Exception {
	SecurityConfiguration sc = new SecurityConfiguration(
		new Configuration(),
		Collections.singletonList(TestSecurityContextFactory.class.getCanonicalName()),
		Collections.singletonList(TestSecurityModuleFactory.class.getCanonicalName()));

	SecurityUtils.install(sc);
	assertEquals(TestSecurityContextFactory.TestSecurityContext.class, SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());
}
 
Example 27
Source Project: flink   Source File: ProcessMemoryUtils.java    License: Apache License 2.0 5 votes vote down vote up
public CommonProcessMemorySpec<FM> memoryProcessSpecFromConfig(Configuration config) {
	if (options.getRequiredFineGrainedOptions().stream().allMatch(config::contains)) {
		// all internal memory options are configured, use these to derive total Flink and process memory
		return deriveProcessSpecWithExplicitInternalMemory(config);
	} else if (config.contains(options.getTotalFlinkMemoryOption())) {
		// internal memory options are not configured, total Flink memory is configured,
		// derive from total flink memory
		return deriveProcessSpecWithTotalFlinkMemory(config);
	} else if (config.contains(options.getTotalProcessMemoryOption())) {
		// total Flink memory is not configured, total process memory is configured,
		// derive from total process memory
		return deriveProcessSpecWithTotalProcessMemory(config);
	}
	return failBecauseRequiredOptionsNotConfigured();
}
 
Example 28
Source Project: flink   Source File: MetricRegistryConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
public static MetricRegistryConfiguration defaultMetricRegistryConfiguration() {
	// create the default metric registry configuration only once
	if (defaultConfiguration == null) {
		synchronized (MetricRegistryConfiguration.class) {
			if (defaultConfiguration == null) {
				defaultConfiguration = fromConfiguration(new Configuration());
			}
		}
	}

	return defaultConfiguration;
}
 
Example 29
Source Project: flink   Source File: FilterITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration config) {
	Collection<Integer> ints = this.getRuntimeContext().getBroadcastVariable("ints");
	for (int i: ints) {
		literal = literal < i ? i : literal;
	}
}
 
Example 30
Source Project: flink   Source File: StreamCheckpointingITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws IOException {
	step = getRuntimeContext().getNumberOfParallelSubtasks();
	if (index == 0) {
		index = getRuntimeContext().getIndexOfThisSubtask();
	}
}