Java Code Examples for org.apache.flink.api.common.functions.RuntimeContext#getIndexOfThisSubtask()

public void open(RuntimeContext context) {
	if (context instanceof StreamingRuntimeContext) {
		this.subtaskId = ((StreamingRuntimeContext) context).getOperatorUniqueID()
			+ "-" + context.getIndexOfThisSubtask();
	} else {
		this.subtaskId = context.getTaskNameWithSubtasks();
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    RuntimeContext context = getRuntimeContext();
    _parallelism = context.getNumberOfParallelSubtasks();
    _partition = context.getIndexOfThisSubtask() + 1;
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext context = getRuntimeContext();
	format.configure(parameters);
	int indexInSubtaskGroup = context.getIndexOfThisSubtask();
	int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
	format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
}
 

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

public void open(RuntimeContext context) {
	if (context instanceof StreamingRuntimeContext) {
		this.subtaskId = ((StreamingRuntimeContext) context).getOperatorUniqueID()
			+ "-" + context.getIndexOfThisSubtask();
	} else {
		this.subtaskId = context.getTaskNameWithSubtasks();
	}
}
 

@Override
public void run(SourceContext<Tuple2<Long, IntType>> ctx) throws Exception {

	final RuntimeContext runtimeContext = getRuntimeContext();
	// detect if this task is "the chosen one" and should fail (via subtaskidx), if it did not fail before (via attempt)
	final boolean failThisTask =
		runtimeContext.getAttemptNumber() == 0 && runtimeContext.getIndexOfThisSubtask() == 0;

	// we loop longer than we have elements, to permit delayed checkpoints
	// to still cause a failure
	while (running && emitCallCount < expectedEmitCalls) {

		// the function failed before, or we are in the elements before the failure
		synchronized (ctx.getCheckpointLock()) {
			eventEmittingGenerator.emitEvent(ctx, emitCallCount++);
		}

		if (emitCallCount < failureAfterNumElements) {
			Thread.sleep(1);
		} else if (failThisTask && emitCallCount == failureAfterNumElements) {
			// wait for a pending checkpoint that fulfills our requirements if needed
			while (checkpointStatus.get() != STATEFUL_CHECKPOINT_COMPLETED) {
				Thread.sleep(1);
			}
			throw new Exception("Artificial Failure");
		}
	}

	if (usingProcessingTime) {
		while (running) {
			Thread.sleep(10);
		}
	}
}
 

@Override
public void run(SourceContext<Long> sourceCtx) throws Exception {
	final Object checkpointLock = sourceCtx.getCheckpointLock();

	RuntimeContext runtimeCtx = getRuntimeContext();

	final long stepSize = runtimeCtx.getNumberOfParallelSubtasks();
	final long congruence = runtimeCtx.getIndexOfThisSubtask();
	final long toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);

	final File proceedFile = new File(coordinateDir, PROCEED_MARKER_FILE);
	boolean checkForProceedFile = true;

	while (isRunning && collected < toCollect) {
		// check if the proceed file exists (then we go full speed)
		// if not, we always recheck and sleep
		if (checkForProceedFile) {
			if (proceedFile.exists()) {
				checkForProceedFile = false;
			} else {
				// otherwise wait so that we make slow progress
				Thread.sleep(SLEEP_TIME);
			}
		}

		synchronized (checkpointLock) {
			sourceCtx.collect(collected * stepSize + congruence);
			collected++;
		}
	}
}
 

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

public void open(RuntimeContext context) {
	if (context instanceof StreamingRuntimeContext) {
		this.subtaskId = ((StreamingRuntimeContext) context).getOperatorUniqueID()
			+ "-" + context.getIndexOfThisSubtask();
	} else {
		this.subtaskId = context.getTaskNameWithSubtasks();
	}
}
 

@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext context = getRuntimeContext();
	format.configure(parameters);
	int indexInSubtaskGroup = context.getIndexOfThisSubtask();
	int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
	format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
}
 

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext context = getRuntimeContext();
	format.configure(parameters);
	int indexInSubtaskGroup = context.getIndexOfThisSubtask();
	int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
	format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
}
 

@Override
public void run(SourceContext<Long> sourceCtx) throws Exception {
	final Object checkpointLock = sourceCtx.getCheckpointLock();

	RuntimeContext runtimeCtx = getRuntimeContext();

	final long stepSize = runtimeCtx.getNumberOfParallelSubtasks();
	final long congruence = runtimeCtx.getIndexOfThisSubtask();
	final long toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);

	final File proceedFile = new File(coordinateDir, PROCEED_MARKER_FILE);
	boolean checkForProceedFile = true;

	while (isRunning && collected < toCollect) {
		// check if the proceed file exists (then we go full speed)
		// if not, we always recheck and sleep
		if (checkForProceedFile) {
			if (proceedFile.exists()) {
				checkForProceedFile = false;
			} else {
				// otherwise wait so that we make slow progress
				Thread.sleep(SLEEP_TIME);
			}
		}

		synchronized (checkpointLock) {
			sourceCtx.collect(collected * stepSize + congruence);
			collected++;
		}
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    RuntimeContext context = getRuntimeContext();
    _parallelism = context.getNumberOfParallelSubtasks();
    _partition = context.getIndexOfThisSubtask() + 1;

    // Get a shorter name. So FetchUrlsFunction -> (Select fetch status, ...
    // becomes FetchUrlsFunction.
    String taskName = context.getTaskName().replaceFirst(" -> .+", "");
    _executor = new ThreadedExecutor("Flink-crawler-" + taskName, _threadCount);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	RuntimeContext runtimeContext = getRuntimeContext();
	indexOfSubtask = runtimeContext.getIndexOfThisSubtask();
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	RuntimeContext runtimeContext = getRuntimeContext();
	indexOfSubtask = runtimeContext.getIndexOfThisSubtask();
}
 

@Override
public String toString() {
	RuntimeContext ctx = getRuntimeContext();

	return ctx.getTaskName() + " (" + ctx.getIndexOfThisSubtask() + "/" + ctx.getNumberOfParallelSubtasks() + ")";
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	final RuntimeContext runtimeContext = getRuntimeContext();
	final int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	final int parallelism = runtimeContext.getNumberOfParallelSubtasks();
	final int maxParallelism = runtimeContext.getMaxNumberOfParallelSubtasks();

	ListStateDescriptor<Long> unionWatermarksStateDescriptor =
		new ListStateDescriptor<>("watermarks", Long.class);

	lastEventTimes = context.getOperatorStateStore().getUnionListState(unionWatermarksStateDescriptor);

	ListStateDescriptor<KeyRangeStates> stateDescriptor =
		new ListStateDescriptor<>("keyRanges", KeyRangeStates.class);

	snapshotKeyRanges = context.getOperatorStateStore().getListState(stateDescriptor);
	keyRanges = new ArrayList<>();

	if (context.isRestored()) {
		// restore key ranges from the snapshot
		for (KeyRangeStates keyRange : snapshotKeyRanges.get()) {
			keyRanges.add(keyRange);
		}

		// let event time start from the max of all event time progress across subtasks in the last execution
		for (Long lastEventTime : lastEventTimes.get()) {
			monotonousEventTime = Math.max(monotonousEventTime, lastEventTime);
		}
	} else {
		// determine the key ranges that belong to the subtask
		int rangeStartIdx = (subtaskIdx * maxParallelism) / parallelism;
		int rangeEndIdx = ((subtaskIdx + 1) * maxParallelism) / parallelism;

		for (int i = rangeStartIdx; i < rangeEndIdx; ++i) {

			int start = ((i * totalKeySpaceSize + maxParallelism - 1) / maxParallelism);
			int end = 1 + ((i + 1) * totalKeySpaceSize - 1) / maxParallelism;

			if (end - start > 0) {
				keyRanges.add(new KeyRangeStates(start, end));
			}
		}

		// fresh run; start from event time o
		monotonousEventTime = 0L;
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext ctx = getRuntimeContext();
	parallelism = ctx.getNumberOfParallelSubtasks();
	counter = ctx.getIndexOfThisSubtask();
}