org.apache.flink.api.common.accumulators.LongCounter Java Examples

The following examples show how to use org.apache.flink.api.common.accumulators.LongCounter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: JavaCounterApp.java From 163-bigdate-note with GNU General Public License v3.0

6 votes

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSource<String> data = env.fromElements("hadoop", "spark", "flink", "strom", "pyspark");

    data.map(new RichMapFunction<String, String>() {
        LongCounter counter = new LongCounter();

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            getRuntimeContext().addAccumulator("ele_counter_java", counter);
        }

        @Override
        public String map(String value) throws Exception {
            counter.add(1);
            return value;
        }
    }).writeAsText("file:\\D:\\imooc\\新一代大数据计算引擎 Flink从入门到实战-v\\input\\sinkout\\sink-java-counter.txt",
            FileSystem.WriteMode.OVERWRITE).setParallelism(3);

    JobExecutionResult counterApp = env.execute("JavaCounterApp");
    Long num = counterApp.getAccumulatorResult("ele_counter_java");
    System.out.println("num:" + num);
}

Example #2

Source File: EdgeMetrics.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #3

Source File: TriadicCensus.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}

Example #4

Source File: MiscellaneousIssuesITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #5

Source File: FlinkFactDistinctColumns.java From kylin with Apache License 2.0

5 votes

public FlatOutputMapPartitionFunction(SerializableConfiguration conf, String cubeName, String segmentId,
                                      String metaUrl, int samplingPercentage, String bytesWrittenName,
                                      String recordCounterName) {
    this.cubeName = cubeName;
    this.segmentId = segmentId;
    this.metaUrl = metaUrl;
    this.conf = conf;
    this.samplingPercentage = samplingPercentage;
    this.bytesWrittenName = bytesWrittenName;
    this.recordCounterName = recordCounterName;
    this.bytesWrittenCounter = new LongCounter();
    this.recordCounter = new LongCounter();
}

Example #6

Source File: MiscellaneousIssuesITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #7

Source File: VertexMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #8

Source File: EdgeMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #9

Source File: VertexMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #10

Source File: TriadicCensus.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}

Example #11

Source File: EdgeMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #12

Source File: EdgeMetrics.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #13

Source File: VertexMetrics.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #14

Source File: VertexMetrics.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #15

Source File: VertexMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #16

Source File: TriadicCensus.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}

Example #17

Source File: EdgeMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #18

Source File: VertexMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}

Example #19

Source File: EdgeMetrics.java From flink with Apache License 2.0

5 votes

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}

Example #20

Source File: FlinkFlowProcess.java From cascading-flink with Apache License 2.0

5 votes

@Override
public void increment(String group, String counter, long l) {
	if(this.runtimeContext != null) {
		LongCounter flinkCounter = getOrInitCounter(EnumStringConverter.mergeGroupCounter(group, counter));
		flinkCounter.add(l);
	}
}

Example #21

Source File: CounterUtils.java From flink-crawler with Apache License 2.0

5 votes

public static void increment(RuntimeContext runtimeContext, String group, String counter,
        long l) {
    if (runtimeContext != null) {
        LongCounter flinkCounter = getOrInitCounter(runtimeContext,
                mergeGroupCounter(group, counter));
        flinkCounter.add(l);
    }
}

Example #22

Source File: MiscellaneousIssuesITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #23

Source File: AccumulatorErrorITCase.java From flink with Apache License 2.0

4 votes

@Override
public LongCounter clone() {
	throw new CustomException();
}

Example #24

Source File: Count.java From flink with Apache License 2.0

4 votes

@Override
public void close() throws IOException {
	addAccumulator(COUNT, new LongCounter(count));
}

Example #25

Source File: ReductionsTest.java From flink-statefun with Apache License 2.0

4 votes

@Override
public LongCounter getLongCounter(String name) {
  throw new UnsupportedOperationException();
}

Example #26

Source File: AccumulatorErrorITCase.java From flink with Apache License 2.0

4 votes

@Override
public void open(Configuration parameters) throws Exception {
	getRuntimeContext().addAccumulator(INCOMPATIBLE_ACCUMULATORS_NAME, new LongCounter());
}

Example #27

Source File: AccumulatorErrorITCase.java From flink with Apache License 2.0

4 votes

@Override
public LongCounter clone() {
	return new FaultyMergeAccumulator();
}

Example #28

Source File: AbstractRuntimeUDFContext.java From flink with Apache License 2.0

4 votes

@Override
public LongCounter getLongCounter(String name) {
	return (LongCounter) getAccumulator(name, LongCounter.class);
}

Example #29

Source File: CepRuntimeContext.java From flink with Apache License 2.0

4 votes

@Override
public LongCounter getLongCounter(final String name) {
	throw new UnsupportedOperationException("Long counters are not supported.");
}

Example #30

Source File: GrepJob.java From flink-perf with Apache License 2.0

4 votes

public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	String in = args[0];
	String out = args[1];
	System.err.println("Using input=" + in);
	System.err.println("Using output=" + out);

	String patterns[] = new String[args.length - 2];
	System.arraycopy(args, 2, patterns, 0, args.length - 2);
	System.err.println("Using patterns: " + Arrays.toString(patterns));

	// get input data
	DataSet<String> text = env.readTextFile(args[0]);
	for (int p = 0; p < patterns.length; p++) {
		final String pattern = patterns[p];
		DataSet<String> res = text.filter(new RichFilterFunction<String>() {
			private static final long serialVersionUID = 1L;

			Pattern p = Pattern.compile(pattern);
			LongCounter filterMatches = new LongCounter();
			LongCounter filterRecords = new LongCounter();

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);
				getRuntimeContext().addAccumulator("filterMatchCount-" + pattern, filterMatches);
				getRuntimeContext().addAccumulator("filterRecordCount-" + pattern, filterRecords);
			}

			@Override
			public boolean filter(String value) throws Exception {
				filterRecords.add(1L);
				if (value == null || value.length() == 0) {
					return false;
				}
				final Matcher m = p.matcher(value);
				if (m.find()) {
					filterMatches.add(1L);
					return true;
				}
				return false;
			}
		}).name("grep for " + pattern);
		res.writeAsText(out + "_" + pattern, FileSystem.WriteMode.OVERWRITE);
	}

	// execute program
	JobExecutionResult jobResult = env.execute("Flink Grep benchmark");
	System.err.println(AccumulatorHelper.getResultsFormated(jobResult.getAllAccumulatorResults()));
}