Java Code Examples for org.apache.flink.api.common.accumulators.LongCounter

The following examples show how to use org.apache.flink.api.common.accumulators.LongCounter. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: 163-bigdate-note   Author: jiaoqiyuan   File: JavaCounterApp.java    License: GNU General Public License v3.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSource<String> data = env.fromElements("hadoop", "spark", "flink", "strom", "pyspark");

    data.map(new RichMapFunction<String, String>() {
        LongCounter counter = new LongCounter();

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            getRuntimeContext().addAccumulator("ele_counter_java", counter);
        }

        @Override
        public String map(String value) throws Exception {
            counter.add(1);
            return value;
        }
    }).writeAsText("file:\\D:\\imooc\\新一代大数据计算引擎 Flink从入门到实战-v\\input\\sinkout\\sink-java-counter.txt",
            FileSystem.WriteMode.OVERWRITE).setParallelism(3);

    JobExecutionResult counterApp = env.execute("JavaCounterApp");
    Long num = counterApp.getAccumulatorResult("ele_counter_java");
    System.out.println("num:" + num);
}
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: MiscellaneousIssuesITCase.java    License: Apache License 2.0 5 votes vote down vote up 
@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: TriadicCensus.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #8
Source Project: flink   Author: flink-tpc-ds   File: MiscellaneousIssuesITCase.java    License: Apache License 2.0 5 votes vote down vote up 
@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #9
Source Project: flink   Author: flink-tpc-ds   File: TriadicCensus.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
Example #10
Source Project: flink   Author: flink-tpc-ds   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #11
Source Project: flink   Author: flink-tpc-ds   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #12
Source Project: flink   Author: flink-tpc-ds   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #13
Source Project: flink   Author: flink-tpc-ds   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #14
Source Project: flink-crawler   Author: ScaleUnlimited   File: CounterUtils.java    License: Apache License 2.0 5 votes vote down vote up 
public static void increment(RuntimeContext runtimeContext, String group, String counter,
        long l) {
    if (runtimeContext != null) {
        LongCounter flinkCounter = getOrInitCounter(runtimeContext,
                mergeGroupCounter(group, counter));
        flinkCounter.add(l);
    }
}
Example #15
Source Project: cascading-flink   Author: dataArtisans   File: FlinkFlowProcess.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void increment(String group, String counter, long l) {
	if(this.runtimeContext != null) {
		LongCounter flinkCounter = getOrInitCounter(EnumStringConverter.mergeGroupCounter(group, counter));
		flinkCounter.add(l);
	}
}
Example #16
Source Project: kylin   Author: apache   File: FlinkFactDistinctColumns.java    License: Apache License 2.0 5 votes vote down vote up 
public FlatOutputMapPartitionFunction(SerializableConfiguration conf, String cubeName, String segmentId,
                                      String metaUrl, int samplingPercentage, String bytesWrittenName,
                                      String recordCounterName) {
    this.cubeName = cubeName;
    this.segmentId = segmentId;
    this.metaUrl = metaUrl;
    this.conf = conf;
    this.samplingPercentage = samplingPercentage;
    this.bytesWrittenName = bytesWrittenName;
    this.recordCounterName = recordCounterName;
    this.bytesWrittenCounter = new LongCounter();
    this.recordCounter = new LongCounter();
}
Example #17
Source Project: flink   Author: apache   File: MiscellaneousIssuesITCase.java    License: Apache License 2.0 5 votes vote down vote up 
@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #18
Source Project: flink   Author: apache   File: TriadicCensus.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
Example #19
Source Project: flink   Author: apache   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #20
Source Project: flink   Author: apache   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #21
Source Project: flink   Author: apache   File: VertexMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
Example #22
Source Project: flink   Author: apache   File: EdgeMetrics.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: AccumulatorErrorITCase.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public LongCounter clone() {
	throw new CustomException();
}
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: AccumulatorErrorITCase.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public void open(Configuration parameters) throws Exception {
	getRuntimeContext().addAccumulator(INCOMPATIBLE_ACCUMULATORS_NAME, new LongCounter());
}
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: AccumulatorErrorITCase.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public LongCounter clone() {
	return new FaultyMergeAccumulator();
}
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractRuntimeUDFContext.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public LongCounter getLongCounter(String name) {
	return (LongCounter) getAccumulator(name, LongCounter.class);
}
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: CepRuntimeContext.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public LongCounter getLongCounter(final String name) {
	throw new UnsupportedOperationException("Long counters are not supported.");
}
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: Count.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(COUNT, new LongCounter(count));
}
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: AverageClusteringCoefficient.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(SUM_OF_LOCAL_CLUSTERING_COEFFICIENT, new DoubleCounter(sumOfLocalClusteringCoefficient));
}
Example #30
Source Project: Flink-CEPplus   Author: ljygz   File: AverageClusteringCoefficient.java    License: Apache License 2.0 4 votes vote down vote up 
@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(SUM_OF_LOCAL_CLUSTERING_COEFFICIENT, new DoubleCounter(sumOfLocalClusteringCoefficient));
}