org.apache.flink.configuration.Configuration Java Examples

@Test
public void testCreateProgramOptionsWithLongOptions() throws CliArgsException {
	String[] args = {
		"--python", "xxx.py",
		"--pyModule", "xxx",
		"--pyFiles", "/absolute/a.py,relative/b.py,relative/c.py",
		"--pyRequirements", "d.txt#e_dir",
		"--pyExecutable", "/usr/bin/python",
		"--pyArchives", "g.zip,h.zip#data,h.zip#data2",
		"userarg1", "userarg2"
	};

	CommandLine line = CliFrontendParser.parse(options, args, false);
	PythonProgramOptions programOptions = (PythonProgramOptions) ProgramOptions.create(line);
	Configuration config = new Configuration();
	programOptions.applyToConfiguration(config);
	assertEquals("/absolute/a.py,relative/b.py,relative/c.py", config.get(PythonOptions.PYTHON_FILES));
	assertEquals("d.txt#e_dir", config.get(PYTHON_REQUIREMENTS));
	assertEquals("g.zip,h.zip#data,h.zip#data2", config.get(PythonOptions.PYTHON_ARCHIVES));
	assertEquals("/usr/bin/python", config.get(PYTHON_EXECUTABLE));
	assertArrayEquals(
		new String[] {"--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2"},
		programOptions.getProgramArgs());
}
 

/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 

private void launchMultiExecuteJob(final boolean enforceSingleJobExecution) throws ProgramInvocationException {
	try (final ClusterClient<?> clusterClient =
				new MiniClusterClient(new Configuration(), MINI_CLUSTER_RESOURCE.getMiniCluster())) {

		final PackagedProgram program = PackagedProgram.newBuilder()
				.setEntryPointClassName(TestMultiExecute.class.getName())
				.build();

		final Configuration configuration = fromPackagedProgram(program, 1, false);

		ClientUtils.executeProgram(
				new TestExecutorServiceLoader(clusterClient, plan),
				configuration,
				program,
				enforceSingleJobExecution,
				false);
	}
}
 

public SlsRecordReader(
		String endPoint,
		Configuration properties,
		String project,
		String logStore,
		int startInSec,
		int stopInSec,
		int maxRetryTime,
		int batchGetSize,
		List<Shard> initShardList,
		String consumerGroup) {
	this.endPoint = endPoint;
	this.project = project;
	this.logStore = logStore;
	this.startInSec = startInSec;
	this.stopInSec = stopInSec;
	this.maxRetryTime = maxRetryTime;
	this.batchGetSize = batchGetSize;
	setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
	this.properties = properties;
	this.consumerGroup = consumerGroup;
}
 

protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 

private TaskDeploymentDescriptor createTestTaskDeploymentDescriptor(
	String taskName,
	ExecutionAttemptID eid,
	Class<? extends AbstractInvokable> abstractInvokable,
	int maxNumberOfSubtasks,
	Collection<ResultPartitionDeploymentDescriptor> producedPartitions,
	Collection<InputGateDeploymentDescriptor> inputGates
) throws IOException {
	Preconditions.checkNotNull(producedPartitions);
	Preconditions.checkNotNull(inputGates);
	return createTaskDeploymentDescriptor(
		jobId, testName.getMethodName(), eid,
		new SerializedValue<>(new ExecutionConfig()), taskName, maxNumberOfSubtasks, 0, 1, 0,
		new Configuration(), new Configuration(), abstractInvokable.getName(),
		producedPartitions,
		inputGates,
		Collections.emptyList(),
		Collections.emptyList(),
		0);
}
 

@Test
public void testDefaultFsParameterLoading() throws Exception {
	try {
		final File tmpDir = temporaryFolder.newFolder();
		final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);

		final URI defaultFS = new URI("otherFS", null, "localhost", 1234, null, null, null);

		final PrintWriter pw1 = new PrintWriter(confFile);
		pw1.println("fs.default-scheme: " + defaultFS);
		pw1.close();

		String[] args = new String[] {"--configDir", tmpDir.toString()};
		Configuration configuration = TaskManagerRunner.loadConfiguration(args);
		FileSystem.initialize(configuration);

		assertEquals(defaultFS, FileSystem.getDefaultFsUri());
	}
	finally {
		// reset FS settings
		FileSystem.initialize(new Configuration());
	}
}
 

protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	this.taskId = getRuntimeContext().getIndexOfThisSubtask();

	LOG.info("{} open.", getRuntimeContext().getTaskName());

	List<Tuple2<Integer, Long>> allCnt = getRuntimeContext().getBroadcastVariable("cnt");

	for (Tuple2<Integer, Long> localCnt : allCnt) {
		if (localCnt.f0 == taskId) {
			cnt = localCnt.f1.intValue();
			break;
		}
	}
}
 

private static void initializeActorSystem(String hostname) throws UnknownHostException {
  synchronized (actorSystemLock) {
    if (actorSystem == null) {
      Configuration config = new Configuration();
      Option<scala.Tuple2<String, Object>> remoting = new Some<>(new scala.Tuple2<String, Object>(hostname, 0));

      Config akkaConfig = AkkaUtils.getAkkaConfig(config, remoting);

      LOG.info("Start actory system.");
      actorSystem = ActorSystem.create("queryableWindow", akkaConfig);
      actorSystemUsers = 1;
    } else {
      LOG.info("Actor system has already been started.");
      actorSystemUsers++;
    }
  }
}
 

public static void main(String[] args) throws Exception {
			Configuration config = new Configuration();

	config.setString(AkkaOptions.ASK_TIMEOUT, "5 s");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> data = env.createInput(new CustomInputFormat());

	data.map(new MapFunction<Integer, Tuple2<Integer, Double>>() {
		@Override
		public Tuple2<Integer, Double> map(Integer value) throws Exception {
			return new Tuple2<Integer, Double>(value, value * 0.5);
		}
	}).addSink(new DiscardingSink<>());

	env.execute();
}
 

@Test
public void testFailedToInitWithInvalidSslKeystoreConfigured() {
	final Configuration config = new Configuration();

	config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true);
	config.setString(SecurityOptions.SSL_KEYSTORE, "invalid.keystore");
	config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_TRUSTSTORE, "invalid.keystore");
	config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password");

	try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) {
		fail();
	} catch (Exception e) {
		findThrowable(e, IOException.class);
		findThrowableWithMessage(e, "Failed to initialize SSL for the blob server");
	}
}
 

@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 

@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 

private Configuration getConfiguration() {
	final Inet6Address ipv6address = getLocalIPv6Address();
	if (ipv6address == null) {
		throw new AssumptionViolatedException("--- Cannot find a non-loopback local IPv6 address that Akka/Netty can bind to; skipping IPv6HostnamesITCase");
	}
	final String addressString = ipv6address.getHostAddress();
	log.info("Test will use IPv6 address " + addressString + " for connection tests");

	Configuration config = new Configuration();
	config.setString(JobManagerOptions.ADDRESS, addressString);
	config.setString(TaskManagerOptions.HOST, addressString);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m");
	return config;
}
 

protected static Collection<Object[]> toParameterList(Configuration ... testConfigs) {
	ArrayList<Object[]> configs = new ArrayList<>();
	for (Configuration testConfig : testConfigs) {
		Object[] c = { testConfig };
		configs.add(c);
	}
	return configs;
}
 

@Test
@SuppressWarnings("unchecked")
public void testFetcherShouldNotBeRestoringFromFailureIfNotRestoringFromCheckpoint() throws Exception {
	KinesisDataFetcher mockedFetcher = mockKinesisDataFetcher();

	// assume the given config is correct
	PowerMockito.mockStatic(KinesisConfigUtil.class);
	PowerMockito.doNothing().when(KinesisConfigUtil.class);

	TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
		"fakeStream", new Properties(), 10, 2);
	consumer.open(new Configuration());
	consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
}
 

public static RetryingRegistrationConfiguration fromConfiguration(final Configuration configuration) {
	long initialRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT);
	long maxRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT);
	long errorDelayMillis = configuration.getLong(ClusterOptions.ERROR_REGISTRATION_DELAY);
	long refusedDelayMillis = configuration.getLong(ClusterOptions.REFUSED_REGISTRATION_DELAY);

	return new RetryingRegistrationConfiguration(
		initialRegistrationTimeoutMillis,
		maxRegistrationTimeoutMillis,
		errorDelayMillis,
		refusedDelayMillis);
}
 

public TestProcessBuilder addConfigAsMainClassArgs(Configuration config) {
	for (Entry<String, String> keyValue: config.toMap().entrySet()) {
		addMainClassArg("--" + keyValue.getKey());
		addMainClassArg(keyValue.getValue());
	}
	return this;
}
 

@Test
public void testUriParametersDefault() throws Exception {
	Configuration config = new Configuration();

	MesosTaskManagerParameters params = MesosTaskManagerParameters.create(config);
	assertEquals(params.uris().size(), 0);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	if (closeCalled) {
		Assert.fail("Close called before open.");
	}
	openCalled = true;
}
 

@Test
public void readWithParseQuotedStrings() {
	try {
		final String fileContent = "\"ab\\\"c\"|\"def\"\n\"ghijk\"|\"abc\"";
		final FileInputSplit split = createTempFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class);
		format.enableQuotedStringParsing('"');

		format.configure(parameters);
		format.open(split);

		Value[] values = new Value[] { new StringValue(), new StringValue()};

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ab\\\"c", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ghijk", ((StringValue) values[0]).getValue());
		assertEquals("abc", ((StringValue) values[1]).getValue());

	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testSecurityContext() throws Exception {
	SecurityConfiguration sc = new SecurityConfiguration(
		new Configuration(),
		Collections.singletonList(TestSecurityContextFactory.class.getCanonicalName()),
		Collections.singletonList(TestSecurityModuleFactory.class.getCanonicalName()));

	SecurityUtils.install(sc);
	assertEquals(TestSecurityContextFactory.TestSecurityContext.class, SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());
}
 

public CommonProcessMemorySpec<FM> memoryProcessSpecFromConfig(Configuration config) {
	if (options.getRequiredFineGrainedOptions().stream().allMatch(config::contains)) {
		// all internal memory options are configured, use these to derive total Flink and process memory
		return deriveProcessSpecWithExplicitInternalMemory(config);
	} else if (config.contains(options.getTotalFlinkMemoryOption())) {
		// internal memory options are not configured, total Flink memory is configured,
		// derive from total flink memory
		return deriveProcessSpecWithTotalFlinkMemory(config);
	} else if (config.contains(options.getTotalProcessMemoryOption())) {
		// total Flink memory is not configured, total process memory is configured,
		// derive from total process memory
		return deriveProcessSpecWithTotalProcessMemory(config);
	}
	return failBecauseRequiredOptionsNotConfigured();
}
 

public static MetricRegistryConfiguration defaultMetricRegistryConfiguration() {
	// create the default metric registry configuration only once
	if (defaultConfiguration == null) {
		synchronized (MetricRegistryConfiguration.class) {
			if (defaultConfiguration == null) {
				defaultConfiguration = fromConfiguration(new Configuration());
			}
		}
	}

	return defaultConfiguration;
}
 

@Override
public void open(Configuration config) {
	Collection<Integer> ints = this.getRuntimeContext().getBroadcastVariable("ints");
	for (int i: ints) {
		literal = literal < i ? i : literal;
	}
}
 

@Override
public void open(Configuration parameters) throws IOException {
	step = getRuntimeContext().getNumberOfParallelSubtasks();
	if (index == 0) {
		index = getRuntimeContext().getIndexOfThisSubtask();
	}
}