org.apache.flink.util.TimeUtils Java Examples

The following examples show how to use org.apache.flink.util.TimeUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DescriptorProperties.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Validates a Java {@link Duration}. The boundaries are inclusive and in milliseconds.
 *
 * <p>The precision defines the allowed minimum unit in milliseconds (e.g. 1000 would only allow seconds).
 */
public void validateDuration(String key, boolean isOptional, int precision, long min, long max) {
	Preconditions.checkArgument(precision > 0);

	validateComparable(
		key,
		isOptional,
		min,
		max,
		"time interval (in milliseconds)",
		(value) -> {
			final long ms = TimeUtils.parseDuration(value).toMillis();
			if (ms % precision != 0) {
				throw new ValidationException(
					"Duration for key '" + key + "' must be a multiple of " + precision + " milliseconds but was: " + value);
			}
			return ms;
		}
	);
}
 
Example #2
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaximumRegistrationDuration() throws Exception {
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("10 ms"));

	final TaskExecutor taskExecutor = createTaskExecutor(new TaskManagerServicesBuilder().build());

	taskExecutor.start();

	try {
		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example #3
Source File: ResourceManagerRuntimeServicesConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(
		Configuration configuration,
		WorkerResourceSpecFactory defaultWorkerResourceSpecFactory) throws ConfigurationException {

	final String strJobTimeout = configuration.getString(ResourceManagerOptions.JOB_TIMEOUT);
	final Time jobTimeout;

	try {
		jobTimeout = Time.milliseconds(TimeUtils.parseDuration(strJobTimeout).toMillis());
	} catch (IllegalArgumentException e) {
		throw new ConfigurationException("Could not parse the resource manager's job timeout " +
			"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
	}

	final WorkerResourceSpec defaultWorkerResourceSpec = defaultWorkerResourceSpecFactory.createDefaultWorkerResourceSpec(configuration);
	final SlotManagerConfiguration slotManagerConfiguration =
		SlotManagerConfiguration.fromConfiguration(configuration, defaultWorkerResourceSpec);

	return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
}
 
Example #4
Source File: DescriptorProperties.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Validates a Java {@link Duration}. The boundaries are inclusive and in milliseconds.
 *
 * <p>The precision defines the allowed minimum unit in milliseconds (e.g. 1000 would only allow seconds).
 */
public void validateDuration(String key, boolean isOptional, int precision, long min, long max) {
	Preconditions.checkArgument(precision > 0);

	validateComparable(
		key,
		isOptional,
		min,
		max,
		"time interval (in milliseconds)",
		(value) -> {
			final long ms = TimeUtils.parseDuration(value).toMillis();
			if (ms % precision != 0) {
				throw new ValidationException(
					"Duration for key '" + key + "' must be a multiple of " + precision + " milliseconds but was: " + value);
			}
			return ms;
		}
	);
}
 
Example #5
Source File: ConfigOptionsDocGenerator.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static String stringifyObject(Object value) {
	if (value instanceof String) {
		if (((String) value).isEmpty()) {
			return "(none)";
		}
		return "\"" + value + "\"";
	} else if (value instanceof Duration) {
		return TimeUtils.formatWithHighestUnit((Duration) value);
	} else if (value instanceof List) {
		return ((List<Object>) value).stream()
			.map(ConfigOptionsDocGenerator::stringifyObject)
			.collect(Collectors.joining(";"));
	} else if (value instanceof Map) {
		return ((Map<String, String>) value)
			.entrySet()
			.stream()
			.map(e -> String.format("%s:%s", e.getKey(), e.getValue()))
			.collect(Collectors.joining(","));
	}
	return value == null ? "(none)" : value.toString();
}
 
Example #6
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Duration getConfiguredIntervalOrDefault(ReporterSetup reporterSetup) {
	final Optional<String> configuredPeriod = reporterSetup.getIntervalSettings();
	Duration period = MetricOptions.REPORTER_INTERVAL.defaultValue();

	if (configuredPeriod.isPresent()) {
		try {
			period = TimeUtils.parseDuration(configuredPeriod.get());
		}
		catch (Exception e) {
			LOG.error("Cannot parse report interval from config: " + configuredPeriod +
				" - please use values like '10 SECONDS' or '500 MILLISECONDS'. " +
				"Using default reporting interval.");
		}
	}
	return period;
}
 
Example #7
Source File: Configuration.java    From flink with Apache License 2.0 5 votes vote down vote up
private Duration convertToDuration(Object o) {
	if (o.getClass() == Duration.class) {
		return (Duration) o;
	}

	return TimeUtils.parseDuration(o.toString());
}
 
Example #8
Source File: TaskManagerRunnerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testShouldShutdownIfRegistrationWithJobManagerFails() throws Exception {
	Configuration configuration = createConfiguration();
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("10 ms"));
	taskManagerRunner = createTaskManagerRunner(configuration);

	Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
	assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
}
 
Example #9
Source File: TaskManagerRunnerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testShouldShutdownOnFatalError() throws Exception {
	Configuration configuration = createConfiguration();
	// very high timeout, to ensure that we don't fail because of registration timeouts
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("42 h"));
	taskManagerRunner = createTaskManagerRunner(configuration);

	taskManagerRunner.onFatalError(new RuntimeException());

	Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
	assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
}
 
Example #10
Source File: AkkaRpcService.java    From flink with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<ActorRef> resolveActorAddress(String address) {
	final ActorSelection actorSel = actorSystem.actorSelection(address);

	return actorSel.resolveOne(TimeUtils.toDuration(configuration.getTimeout()))
		.toCompletableFuture()
		.exceptionally(error -> {
			throw new CompletionException(
				new RpcConnectionException(String.format("Could not connect to rpc endpoint under address %s.", address), error));
		});
}
 
Example #11
Source File: DescriptorProperties.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a Java {@link Duration} under the given key if it exists.
 */
public Optional<Duration> getOptionalDuration(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return TimeUtils.parseDuration(value);
		} catch (Exception e) {
			throw new ValidationException("Invalid duration value for key '" + key + "'.", e);
		}
	});
}
 
Example #12
Source File: DescriptorProperties.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a Java {@link Duration} under the given key if it exists.
 */
public Optional<Duration> getOptionalDuration(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return TimeUtils.parseDuration(value);
		} catch (Exception e) {
			throw new ValidationException("Invalid duration value for key '" + key + "'.", e);
		}
	});
}
 
Example #13
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
MetricRegistryImpl(MetricRegistryConfiguration config, Collection<ReporterSetup> reporterConfigurations, ScheduledExecutorService scheduledExecutor) {
	this.maximumFramesize = config.getQueryServiceMessageSizeLimit();
	this.scopeFormats = config.getScopeFormats();
	this.globalDelimiter = config.getDelimiter();
	this.terminationFuture = new CompletableFuture<>();
	this.isShutdown = false;

	// second, instantiate any custom configured reporters
	this.reporters = new ArrayList<>(4);

	this.executor = scheduledExecutor;

	this.queryService = null;
	this.metricQueryServiceRpcService = null;

	if (reporterConfigurations.isEmpty()) {
		// no reporters defined
		// by default, don't report anything
		LOG.info("No metrics reporter configured, no metrics will be exposed/reported.");
	} else {
		for (ReporterSetup reporterSetup : reporterConfigurations) {
			final String namedReporter = reporterSetup.getName();

			try {
				final MetricReporter reporterInstance = reporterSetup.getReporter();
				final String className = reporterInstance.getClass().getName();

				if (reporterInstance instanceof Scheduled) {
					final Duration period = getConfiguredIntervalOrDefault(reporterSetup);

					LOG.info("Periodically reporting metrics in intervals of {} for reporter {} of type {}.", TimeUtils.formatWithHighestUnit(period), namedReporter, className);

					executor.scheduleWithFixedDelay(
							new MetricRegistryImpl.ReporterTask((Scheduled) reporterInstance), period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS);
				} else {
					LOG.info("Reporting metrics for reporter {} of type {}.", namedReporter, className);
				}

				String delimiterForReporter = reporterSetup.getDelimiter().orElse(String.valueOf(globalDelimiter));
				if (delimiterForReporter.length() != 1) {
					LOG.warn("Failed to parse delimiter '{}' for reporter '{}', using global delimiter '{}'.", delimiterForReporter, namedReporter, globalDelimiter);
					delimiterForReporter = String.valueOf(globalDelimiter);
				}

				reporters.add(new ReporterAndSettings(
					reporterInstance,
					new ReporterScopedSettings(
						reporters.size(),
						delimiterForReporter.charAt(0),
						reporterSetup.getExcludedVariables())));
			}
			catch (Throwable t) {
				LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", namedReporter, t);
			}
		}
	}
}
 
Example #14
Source File: HiveTableSource.java    From flink with Apache License 2.0 4 votes vote down vote up
private List<HiveTablePartition> initAllPartitions() {
	List<HiveTablePartition> allHivePartitions = new ArrayList<>();
	// Please note that the following directly accesses Hive metastore, which is only a temporary workaround.
	// Ideally, we need to go thru Catalog API to get all info we need here, which requires some major
	// refactoring. We will postpone this until we merge Blink to Flink.
	try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(new HiveConf(jobConf, HiveConf.class), hiveVersion)) {
		String dbName = tablePath.getDatabaseName();
		String tableName = tablePath.getObjectName();
		List<String> partitionColNames = catalogTable.getPartitionKeys();
		Table hiveTable = client.getTable(dbName, tableName);
		Properties tableProps = HiveReflectionUtils.getTableMetadata(hiveShim, hiveTable);
		String ttlStr = tableProps.getProperty(FileSystemOptions.LOOKUP_JOIN_CACHE_TTL.key());
		hiveTableCacheTTL = ttlStr != null ?
				TimeUtils.parseDuration(ttlStr) :
				FileSystemOptions.LOOKUP_JOIN_CACHE_TTL.defaultValue();
		if (partitionColNames != null && partitionColNames.size() > 0) {
			final String defaultPartitionName = jobConf.get(HiveConf.ConfVars.DEFAULTPARTITIONNAME.varname,
					HiveConf.ConfVars.DEFAULTPARTITIONNAME.defaultStrVal);
			List<Partition> partitions = new ArrayList<>();
			if (remainingPartitions != null) {
				for (Map<String, String> spec : remainingPartitions) {
					partitions.add(client.getPartition(dbName, tableName, partitionSpecToValues(spec, partitionColNames)));
				}
			} else {
				partitions.addAll(client.listPartitions(dbName, tableName, (short) -1));
			}
			for (Partition partition : partitions) {
				HiveTablePartition hiveTablePartition = toHiveTablePartition(
						catalogTable.getPartitionKeys(),
						catalogTable.getSchema().getFieldNames(),
						catalogTable.getSchema().getFieldDataTypes(),
						hiveShim,
						tableProps,
						defaultPartitionName,
						partition);
				allHivePartitions.add(hiveTablePartition);
			}
		} else {
			allHivePartitions.add(new HiveTablePartition(hiveTable.getSd(), tableProps));
		}
	} catch (TException e) {
		throw new FlinkHiveException("Failed to collect all partitions from hive metaStore", e);
	}
	return allHivePartitions;
}
 
Example #15
Source File: FunctionJsonEntity.java    From flink-statefun with Apache License 2.0 4 votes vote down vote up
private static Optional<Duration> optionalStateExpireDuration(JsonNode stateSpecNode) {
  return Selectors.optionalTextAt(stateSpecNode, StateSpecPointers.EXPIRE_DURATION)
      .map(TimeUtils::parseDuration);
}
 
Example #16
Source File: FunctionJsonEntity.java    From flink-statefun with Apache License 2.0 4 votes vote down vote up
private static Optional<Duration> optionalMaxRequestDuration(JsonNode functionNode) {
  return Selectors.optionalTextAt(functionNode, SpecPointers.TIMEOUT)
      .map(TimeUtils::parseDuration);
}
 
Example #17
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testMaximumRegistrationDurationAfterConnectionLoss() throws Exception {
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("100 ms"));
	final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices, new HeartbeatServices(10L, 10L));

	taskExecutor.start();

	final CompletableFuture<ResourceID> registrationFuture = new CompletableFuture<>();
	final OneShotLatch secondRegistration = new OneShotLatch();
	try {
		final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
		testingResourceManagerGateway.setRegisterTaskExecutorFunction(
			taskExecutorRegistration -> {
				if (registrationFuture.complete(taskExecutorRegistration.getResourceId())) {
					return createRegistrationResponse(testingResourceManagerGateway);
				} else {
					secondRegistration.trigger();
					return CompletableFuture.completedFuture(new Decline("Only the first registration should succeed."));
				}
			}
		);
		rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);

		resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), UUID.randomUUID());

		final ResourceID registrationResourceId = registrationFuture.get();

		assertThat(registrationResourceId, equalTo(taskManagerServices.getUnresolvedTaskManagerLocation().getResourceID()));

		secondRegistration.await();

		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}