Java Code Examples for org.apache.flink.graph.Graph#run()

The following examples show how to use org.apache.flink.graph.Graph#run() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: EdgeSourceDegree.java From flink with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, d(s)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// s, t, d(s)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(0)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge source degree");
}

Example 2

Source File: ConnectedComponentsWithRandomisedEdgesITCase.java From flink with Apache License 2.0

6 votes

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
	DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));

	DataSet<Edge<Long, NullValue>> edges = edgeString.map(new EdgeParser());

	DataSet<Vertex<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());

	Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);

	DataSet<Vertex<Long, Long>> result = graph.run(new ConnectedComponents<>(100));

	result.writeAsCsv(resultPath, "\n", " ");
	env.execute();
}

Example 3

Source File: EdgeSourceDegrees.java From flink with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple2<EV, Degrees>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, d(s)
	DataSet<Vertex<K, Degrees>> vertexDegrees = input
		.run(new VertexDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// s, t, d(s)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(0)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge source degrees");
}

Example 4

Source File: TriadicCensus.java From flink with Apache License 2.0

6 votes

@Override
public TriadicCensus<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	triangleCount = new Count<>();

	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setSortTriangleVertices(false)
			.setParallelism(parallelism));

	triangleCount.run(triangles);

	vertexMetrics = new VertexMetrics<K, VV, EV>()
		.setParallelism(parallelism);

	input.run(vertexMetrics);

	return this;
}

Example 5

Source File: EdgeTargetDegree.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(!reduceOnSourceId.get())
			.setParallelism(parallelism));

	// s, t, d(t)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}

Example 6

Source File: EdgeDegreePair.java From flink with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, t, d(s)
	DataSet<Edge<K, Tuple2<EV, LongValue>>> edgeSourceDegrees = input
		.run(new EdgeSourceDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// s, t, (d(s), d(t))
	return edgeSourceDegrees
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeDegreeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}

Example 7

Source File: GlobalClusteringCoefficient.java From flink with Apache License 2.0

6 votes

@Override
public GlobalClusteringCoefficient<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	triangleCount = new Count<>();

	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setSortTriangleVertices(false)
			.setParallelism(parallelism));

	triangleCount.run(triangles);

	vertexMetrics = new VertexMetrics<K, VV, EV>()
		.setParallelism(parallelism);

	input.run(vertexMetrics);

	return this;
}

Example 8

Source File: AverageClusteringCoefficient.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public AverageClusteringCoefficient<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	DataSet<LocalClusteringCoefficient.Result<K>> localClusteringCoefficient = input
		.run(new LocalClusteringCoefficient<K, VV, EV>()
			.setParallelism(parallelism));

	averageClusteringCoefficientHelper = new AverageClusteringCoefficientHelper<>();

	localClusteringCoefficient
		.output(averageClusteringCoefficientHelper)
			.name("Average clustering coefficient");

	return this;
}

Example 9

Source File: EdgeDegreesPair.java From flink with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, t, d(s)
	DataSet<Edge<K, Tuple2<EV, Degrees>>> edgeSourceDegrees = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// t, d(t)
	DataSet<Vertex<K, Degrees>> vertexDegrees = input
		.run(new VertexDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// s, t, (d(s), d(t))
	return edgeSourceDegrees
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeDegreeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}

Example 10

Source File: EdgeTargetDegree.java From flink with Apache License 2.0

6 votes

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(!reduceOnSourceId.get())
			.setParallelism(parallelism));

	// s, t, d(t)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}

Example 11

Source File: EdgeMetrics.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public EdgeMetrics<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	// s, t, (d(s), d(t))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> edgeDegreePair = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId)
			.setParallelism(parallelism));

	// s, d(s), count of (u, v) where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, LongValue, LongValue>> edgeStats = edgeDegreePair
		.map(new EdgeStats<>())
			.setParallelism(parallelism)
			.name("Edge stats")
		.groupBy(0)
		.reduce(new SumEdgeStats<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum edge stats");

	edgeMetricsHelper = new EdgeMetricsHelper<>();

	edgeStats
		.output(edgeMetricsHelper)
			.setParallelism(parallelism)
			.name("Edge metrics");

	return this;
}

Example 12

Source File: PageRank.java From flink with Apache License 2.0

5 votes

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.linkanalysis.PageRank<K, VV, EV>(
				dampingFactor.getValue(),
				iterationConvergence.getValue().iterations,
				iterationConvergence.getValue().convergenceThreshold)
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}

Example 13

Source File: EdgeMetrics.java From flink with Apache License 2.0

5 votes

@Override
public EdgeMetrics<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	// s, t, (d(s), d(t))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> edgeDegreePair = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId)
			.setParallelism(parallelism));

	// s, d(s), count of (u, v) where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, LongValue, LongValue>> edgeStats = edgeDegreePair
		.map(new EdgeStats<>())
			.setParallelism(parallelism)
			.name("Edge stats")
		.groupBy(0)
		.reduce(new SumEdgeStats<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum edge stats");

	edgeMetricsHelper = new EdgeMetricsHelper<>();

	edgeStats
		.output(edgeMetricsHelper)
			.setParallelism(parallelism)
			.name("Edge metrics");

	return this;
}

Example 14

Source File: TriangleListing.java From flink with Apache License 2.0

5 votes

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	int parallelism = this.parallelism.getValue().intValue();

	switch (order.getValue()) {
		case DIRECTED:
			if (computeTriadicCensus.getValue()) {
				triadicCensus = graph
					.run(new org.apache.flink.graph.library.clustering.directed.TriadicCensus<K, VV, EV>()
						.setParallelism(parallelism));
			}

			@SuppressWarnings("unchecked")
			DataSet<PrintableResult> directedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
				.run(new org.apache.flink.graph.library.clustering.directed.TriangleListing<K, VV, EV>()
					.setPermuteResults(permuteResults.getValue())
					.setSortTriangleVertices(sortTriangleVertices.getValue())
					.setParallelism(parallelism));
			return directedResult;

		case UNDIRECTED:
			if (computeTriadicCensus.getValue()) {
				triadicCensus = graph
					.run(new org.apache.flink.graph.library.clustering.undirected.TriadicCensus<K, VV, EV>()
						.setParallelism(parallelism));
			}

			@SuppressWarnings("unchecked")
			DataSet<PrintableResult> undirectedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
				.run(new org.apache.flink.graph.library.clustering.undirected.TriangleListing<K, VV, EV>()
					.setPermuteResults(permuteResults.getValue())
					.setSortTriangleVertices(sortTriangleVertices.getValue())
					.setParallelism(parallelism));
			return undirectedResult;

		default:
			throw new RuntimeException("Unknown order: " + order);
	}
}

Example 15

Source File: AdamicAdar.java From flink with Apache License 2.0

5 votes

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.similarity.AdamicAdar<K, VV, EV>()
			.setMinimumRatio(minRatio.getValue().floatValue())
			.setMinimumScore(minScore.getValue().floatValue())
			.setMirrorResults(mirrorResults.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}

Example 16

Source File: PageRank.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.linkanalysis.PageRank<K, VV, EV>(
				dampingFactor.getValue(),
				iterationConvergence.getValue().iterations,
				iterationConvergence.getValue().convergenceThreshold)
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}

Example 17

Source File: AdamicAdarTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Validate a test where each result has the same values.
 *
 * @param graph input graph
 * @param count number of results
 * @param score result score
 * @param <T> graph ID type
 * @throws Exception on error
 */
private static <T extends CopyableValue<T>> void validate(
		Graph<T, NullValue, NullValue> graph, long count, double score) throws Exception {
	DataSet<Result<T>> aa = graph
		.run(new AdamicAdar<>());

	List<Result<T>> results = aa.collect();

	assertEquals(count, results.size());

	for (Result<T> result : results) {
		assertEquals(score, result.getAdamicAdarScore().getValue(), ACCURACY);
	}
}

Example 18

Source File: LocalClusteringCoefficient.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, w
	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setParallelism(parallelism));

	// u, 1
	DataSet<Tuple2<K, LongValue>> triangleVertices = triangles
		.flatMap(new SplitTriangles<>())
			.name("Split triangle vertices");

	// u, triangle count
	DataSet<Tuple2<K, LongValue>> vertexTriangleCount = triangleVertices
		.groupBy(0)
		.reduce(new CountTriangles<>())
		.setCombineHint(CombineHint.HASH)
			.name("Count triangles")
			.setParallelism(parallelism);

	// u, deg(u)
	DataSet<Vertex<K, LongValue>> vertexDegree = input
		.run(new VertexDegree<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.get())
			.setParallelism(parallelism));

	// u, deg(u), triangle count
	return vertexDegree
		.leftOuterJoin(vertexTriangleCount)
		.where(0)
		.equalTo(0)
		.with(new JoinVertexDegreeWithTriangleCount<>())
			.setParallelism(parallelism)
			.name("Clustering coefficient");
}

Example 19

Source File: TriangleListing.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, bitmask where u < v
	DataSet<Tuple3<K, K, ByteValue>> filteredByID = input
		.getEdges()
		.map(new OrderByID<>())
			.setParallelism(parallelism)
			.name("Order by ID")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by ID");

	// u, v, (deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> pairDegrees = input
		.run(new EdgeDegreesPair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v, bitmask where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, K, ByteValue>> filteredByDegree = pairDegrees
		.map(new OrderByDegree<>())
			.setParallelism(parallelism)
			.name("Order by degree")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by degree");

	// u, v, w, bitmask where (u, v) and (u, w) are edges in graph
	DataSet<Tuple4<K, K, K, ByteValue>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w, bitmask where (u, v), (u, w), and (v, w) are edges in graph
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}

Example 20

Source File: PageRank.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}