Java Code Examples for org.apache.flink.runtime.executiongraph.IOMetrics

The following examples show how to use org.apache.flink.runtime.executiongraph.IOMetrics. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: TaskExecutionState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutionState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 
Example #3
Source Project: flink   Author: apache   File: TaskExecutionState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: TaskIOMetricGroupTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInLocalCounter().inc(100L);
	taskIO.getNumBytesInRemoteCounter().inc(150L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersInLocalCounter().inc(1L);
	taskIO.getNumBuffersInRemoteCounter().inc(2L);
	taskIO.getNumBuffersOutCounter().inc(3L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesInLocal());
	assertEquals(150L, io.getNumBytesInRemote());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(1L, taskIO.getNumBuffersInLocalCounter().getCount());
	assertEquals(2L, taskIO.getNumBuffersInRemoteCounter().getCount());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: TaskIOMetricGroupTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInCounter().inc(100L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersOutCounter().inc(3L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesIn());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
}
 
Example #6
Source Project: flink   Author: apache   File: TaskIOMetricGroupTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInCounter().inc(100L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersOutCounter().inc(3L);
	taskIO.getIdleTimeMsPerSecond().markEvent(2L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesIn());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
	assertEquals(2L, taskIO.getIdleTimeMsPerSecond().getCount());
}
 
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: TaskExecutionState.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics getIOMetrics() {
	return ioMetrics;
}
 
Example #8
Source Project: Flink-CEPplus   Author: ljygz   File: TaskIOMetricGroup.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRateLocal, numBytesInRateRemote, numBytesOutRate);
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: MutableIOMetrics.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesInLocal += ioMetrics.getNumBytesInLocal();
			this.numBytesInRemote += ioMetrics.getNumBytesInRemote();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL) == null){
					this.numBytesInLocalComplete = false;
				}
				else {
					this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE) == null){
					this.numBytesInRemoteComplete = false;
				}
				else {
					this.numBytesInRemote += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInLocalComplete = false;
				this.numBytesInRemoteComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: ArchivedJobGenerationUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static void compareIoMetrics(IOMetrics expectedMetrics, JsonNode writtenMetrics) {
	assertEquals(expectedMetrics.getNumBytesInTotal(), writtenMetrics.get("read-bytes").asLong());
	assertEquals(expectedMetrics.getNumBytesOut(), writtenMetrics.get("write-bytes").asLong());
	assertEquals(expectedMetrics.getNumRecordsIn(), writtenMetrics.get("read-records").asLong());
	assertEquals(expectedMetrics.getNumRecordsOut(), writtenMetrics.get("write-records").asLong());
}
 
Example #11
Source Project: Flink-CEPplus   Author: ljygz   File: ArchivedExecutionBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public ArchivedExecutionBuilder setIOMetrics(IOMetrics ioMetrics) {
	this.ioMetrics = ioMetrics;
	return this;
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutionState.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics getIOMetrics() {
	return ioMetrics;
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: TaskIOMetricGroup.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRate, numBytesOutRate);
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: MutableIOMetrics.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: ArchivedExecutionBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public ArchivedExecutionBuilder setIOMetrics(IOMetrics ioMetrics) {
	this.ioMetrics = ioMetrics;
	return this;
}
 
Example #16
Source Project: flink   Author: apache   File: TaskExecutionState.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics getIOMetrics() {
	return ioMetrics;
}
 
Example #17
Source Project: flink   Author: apache   File: TaskIOMetricGroup.java    License: Apache License 2.0 4 votes vote down vote up
public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRate, numBytesOutRate);
}
 
Example #18
Source Project: flink   Author: apache   File: MutableIOMetrics.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}