org.apache.flink.api.common.functions.BroadcastVariableInitializer Java Examples

@SuppressWarnings("unchecked")
@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {

	// check if we have an initialized version
	Object o = this.initializedBroadcastVars.get(name);
	if (o != null) {
		return (C) o;
	}
	else {
		List<T> uninitialized = (List<T>) this.uninitializedBroadcastVars.remove(name);
		if (uninitialized != null) {
			C result = initializer.initializeBroadcastVariable(uninitialized);
			this.initializedBroadcastVars.put(name, result);
			return result;
		}
		else {
			throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
		}
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	RuntimeContext ctx = getRuntimeContext();

	this.taskId = ctx.getIndexOfThisSubtask();
	this.splitPoints = ctx.getBroadcastVariableWithInitializer(
		"splitPoints",
		new BroadcastVariableInitializer<Tuple2<Object, Integer>, List<Tuple2<Object, Integer>>>() {
			@Override
			public List<Tuple2<Object, Integer>> initializeBroadcastVariable(
				Iterable<Tuple2<Object, Integer>> data) {
				// sort the list by task id to calculate the correct offset
				List<Tuple2<Object, Integer>> sortedData = new ArrayList<>();
				for (Tuple2<Object, Integer> datum : data) {
					sortedData.add(datum);
				}
				sortedData.sort(new SortUtils.PairComparator());
				return sortedData;
			}
		});
	outBuff = new Tuple2<>();
	LOG.info("{} open.", getRuntimeContext().getTaskName());
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	RuntimeContext ctx = getRuntimeContext();

	this.taskId = ctx.getIndexOfThisSubtask();
	this.splitPoints = ctx.getBroadcastVariableWithInitializer(
		"splitPoints",
		new BroadcastVariableInitializer <Tuple2 <Object, Integer>, List <Tuple2 <Object, Integer>>>() {
			@Override
			public List <Tuple2 <Object, Integer>> initializeBroadcastVariable(
				Iterable <Tuple2 <Object, Integer>> data) {
				// sort the list by task id to calculate the correct offset
				List <Tuple2 <Object, Integer>> sortedData = new ArrayList <>();
				for (Tuple2 <Object, Integer> datum : data) {
					sortedData.add(datum);
				}
				Collections.sort(sortedData, new PairComparator());
				return sortedData;
			}
		});
}
 

@SuppressWarnings("unchecked")
@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {

	// check if we have an initialized version
	Object o = this.initializedBroadcastVars.get(name);
	if (o != null) {
		return (C) o;
	}
	else {
		List<T> uninitialized = (List<T>) this.uninitializedBroadcastVars.remove(name);
		if (uninitialized != null) {
			C result = initializer.initializeBroadcastVariable(uninitialized);
			this.initializedBroadcastVars.put(name, result);
			return result;
		}
		else {
			throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
		}
	}
}
 

public C getVariable(BroadcastVariableInitializer<T, C> initializer) {
	if (!materialized) {
		throw new IllegalStateException("The Broadcast Variable has not yet been materialized.");
	}
	if (disposed) {
		throw new IllegalStateException("The Broadcast Variable has been disposed");
	}

	synchronized (references) {
		if (transformed == null) {
			transformed = initializer.initializeBroadcastVariable(data);
			data = null;
		}
		return transformed;
	}
}
 

@SuppressWarnings("unchecked")
@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {

	// check if we have an initialized version
	Object o = this.initializedBroadcastVars.get(name);
	if (o != null) {
		return (C) o;
	}
	else {
		List<T> uninitialized = (List<T>) this.uninitializedBroadcastVars.remove(name);
		if (uninitialized != null) {
			C result = initializer.initializeBroadcastVariable(uninitialized);
			this.initializedBroadcastVars.put(name, result);
			return result;
		}
		else {
			throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
		}
	}
}
 

public C getVariable(BroadcastVariableInitializer<T, C> initializer) {
	if (!materialized) {
		throw new IllegalStateException("The Broadcast Variable has not yet been materialized.");
	}
	if (disposed) {
		throw new IllegalStateException("The Broadcast Variable has been disposed");
	}

	synchronized (references) {
		if (transformed == null) {
			transformed = initializer.initializeBroadcastVariable(data);
			data = null;
		}
		return transformed;
	}
}
 

public C getVariable(BroadcastVariableInitializer<T, C> initializer) {
	if (!materialized) {
		throw new IllegalStateException("The Broadcast Variable has not yet been materialized.");
	}
	if (disposed) {
		throw new IllegalStateException("The Broadcast Variable has been disposed");
	}

	synchronized (references) {
		if (transformed == null) {
			transformed = initializer.initializeBroadcastVariable(data);
			data = null;
		}
		return transformed;
	}
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	Preconditions.checkNotNull(name, "The broadcast variable name must not be null.");
	Preconditions.checkNotNull(initializer, "The broadcast variable initializer must not be null.");

	// check if we have an initialized version
	@SuppressWarnings("unchecked")
	BroadcastVariableMaterialization<T, C> variable = (BroadcastVariableMaterialization<T, C>) this.broadcastVars.get(name);
	if (variable != null) {
		return variable.getVariable(initializer);
	}
	else {
		throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
	}
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	Preconditions.checkNotNull(name, "The broadcast variable name must not be null.");
	Preconditions.checkNotNull(initializer, "The broadcast variable initializer must not be null.");

	// check if we have an initialized version
	@SuppressWarnings("unchecked")
	BroadcastVariableMaterialization<T, C> variable = (BroadcastVariableMaterialization<T, C>) this.broadcastVars.get(name);
	if (variable != null) {
		return variable.getVariable(initializer);
	}
	else {
		throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
	}
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	Preconditions.checkNotNull(name, "The broadcast variable name must not be null.");
	Preconditions.checkNotNull(initializer, "The broadcast variable initializer must not be null.");

	// check if we have an initialized version
	@SuppressWarnings("unchecked")
	BroadcastVariableMaterialization<T, C> variable = (BroadcastVariableMaterialization<T, C>) this.broadcastVars.get(name);
	if (variable != null) {
		return variable.getVariable(initializer);
	}
	else {
		throw new IllegalArgumentException("The broadcast variable with name '" + name + "' has not been set.");
	}
}
 

/**
 * Transform the label column as the
 *
 * @param input
 * @param model
 * @param colIdx
 * @return
 */
public static DataSet<Row> findIndexOfLabel(DataSet<Row> input, DataSet<Object[]> model, final int colIdx) {
	return input
		.map(new RichMapFunction<Row, Row>() {
			Object[] model;

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);

				model = getRuntimeContext().getBroadcastVariableWithInitializer("model",
					new BroadcastVariableInitializer<Object[], Object[]>() {
						@Override
						public Object[] initializeBroadcastVariable(Iterable<Object[]> data) {
							return data.iterator().next();
						}
					}
				);
			}

			@Override
			public Row map(Row value) throws Exception {
				value.setField(colIdx, findIdx(model, value.getField(colIdx)));
				return value;
			}
		})
		.withBroadcastSet(model, "model");
}
 

@Override
public void open(Configuration parameters) throws Exception {
	vocSize = getRuntimeContext().getBroadcastVariableWithInitializer("vocSize",
		new BroadcastVariableInitializer <Long, Integer>() {
			@Override
			public Integer initializeBroadcastVariable(Iterable <Long> data) {
				return data.iterator().next().intValue();
			}
		});
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables can only be used in DataSet programs");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
	String name, BroadcastVariableInitializer<T, C> initializer) {
	return ctx.getBroadcastVariableWithInitializer(name, initializer);
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
		final String name,
		final BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported.");
}
 

/**
 * Method that assigns a unique {@link Long} value to all elements in the input data set. The generated values are
 * consecutive.
 *
 * @param input the input data set
 * @return a data set of tuple 2 consisting of consecutive ids and initial values.
 */
public static <T> DataSet<Tuple2<Long, T>> zipWithIndex(DataSet<T> input) {

	DataSet<Tuple2<Integer, Long>> elementCount = countElementsPerPartition(input);

	return input.mapPartition(new RichMapPartitionFunction<T, Tuple2<Long, T>>() {

		long start = 0;

		@Override
		public void open(Configuration parameters) throws Exception {
			super.open(parameters);

			List<Tuple2<Integer, Long>> offsets = getRuntimeContext().getBroadcastVariableWithInitializer(
					"counts",
					new BroadcastVariableInitializer<Tuple2<Integer, Long>, List<Tuple2<Integer, Long>>>() {
						@Override
						public List<Tuple2<Integer, Long>> initializeBroadcastVariable(Iterable<Tuple2<Integer, Long>> data) {
							// sort the list by task id to calculate the correct offset
							List<Tuple2<Integer, Long>> sortedData = new ArrayList<>();
							for (Tuple2<Integer, Long> datum : data) {
								sortedData.add(datum);
							}
							Collections.sort(sortedData, new Comparator<Tuple2<Integer, Long>>() {
								@Override
								public int compare(Tuple2<Integer, Long> o1, Tuple2<Integer, Long> o2) {
									return o1.f0.compareTo(o2.f0);
								}
							});
							return sortedData;
						}
					});

			// compute the offset for each partition
			for (int i = 0; i < getRuntimeContext().getIndexOfThisSubtask(); i++) {
				start += offsets.get(i).f1;
			}
		}

		@Override
		public void mapPartition(Iterable<T> values, Collector<Tuple2<Long, T>> out) throws Exception {
			for (T value: values) {
				out.collect(new Tuple2<>(start++, value));
			}
		}
	}).withBroadcastSet(elementCount, "counts");
}
 

/**
 * Method that assigns a unique {@link Long} value to all elements in the input data set. The generated values are
 * consecutive.
 *
 * @param input the input data set
 * @return a data set of tuple 2 consisting of consecutive ids and initial values.
 */
public static <T> DataSet<Tuple2<Long, T>> zipWithIndex(DataSet<T> input) {

	DataSet<Tuple2<Integer, Long>> elementCount = countElementsPerPartition(input);

	return input.mapPartition(new RichMapPartitionFunction<T, Tuple2<Long, T>>() {

		long start = 0;

		@Override
		public void open(Configuration parameters) throws Exception {
			super.open(parameters);

			List<Tuple2<Integer, Long>> offsets = getRuntimeContext().getBroadcastVariableWithInitializer(
					"counts",
					new BroadcastVariableInitializer<Tuple2<Integer, Long>, List<Tuple2<Integer, Long>>>() {
						@Override
						public List<Tuple2<Integer, Long>> initializeBroadcastVariable(Iterable<Tuple2<Integer, Long>> data) {
							// sort the list by task id to calculate the correct offset
							List<Tuple2<Integer, Long>> sortedData = new ArrayList<>();
							for (Tuple2<Integer, Long> datum : data) {
								sortedData.add(datum);
							}
							Collections.sort(sortedData, new Comparator<Tuple2<Integer, Long>>() {
								@Override
								public int compare(Tuple2<Integer, Long> o1, Tuple2<Integer, Long> o2) {
									return o1.f0.compareTo(o2.f0);
								}
							});
							return sortedData;
						}
					});

			// compute the offset for each partition
			for (int i = 0; i < getRuntimeContext().getIndexOfThisSubtask(); i++) {
				start += offsets.get(i).f1;
			}
		}

		@Override
		public void mapPartition(Iterable<T> values, Collector<Tuple2<Long, T>> out) throws Exception {
			for (T value: values) {
				out.collect(new Tuple2<>(start++, value));
			}
		}
	}).withBroadcastSet(elementCount, "counts");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported in rich async functions.");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
    String name, BroadcastVariableInitializer<T, C> initializer) {
  throw new UnsupportedOperationException();
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
    String name, BroadcastVariableInitializer<T, C> initializer) {
  throw new UnsupportedOperationException();
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported in rich async functions.");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables can only be used in DataSet programs");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
	String name, BroadcastVariableInitializer<T, C> initializer) {
	return ctx.getBroadcastVariableWithInitializer(name, initializer);
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
		final String name,
		final BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported.");
}
 

/**
 * Method that assigns a unique {@link Long} value to all elements in the input data set. The generated values are
 * consecutive.
 *
 * @param input the input data set
 * @return a data set of tuple 2 consisting of consecutive ids and initial values.
 */
public static <T> DataSet<Tuple2<Long, T>> zipWithIndex(DataSet<T> input) {

	DataSet<Tuple2<Integer, Long>> elementCount = countElementsPerPartition(input);

	return input.mapPartition(new RichMapPartitionFunction<T, Tuple2<Long, T>>() {

		long start = 0;

		@Override
		public void open(Configuration parameters) throws Exception {
			super.open(parameters);

			List<Tuple2<Integer, Long>> offsets = getRuntimeContext().getBroadcastVariableWithInitializer(
					"counts",
					new BroadcastVariableInitializer<Tuple2<Integer, Long>, List<Tuple2<Integer, Long>>>() {
						@Override
						public List<Tuple2<Integer, Long>> initializeBroadcastVariable(Iterable<Tuple2<Integer, Long>> data) {
							// sort the list by task id to calculate the correct offset
							List<Tuple2<Integer, Long>> sortedData = new ArrayList<>();
							for (Tuple2<Integer, Long> datum : data) {
								sortedData.add(datum);
							}
							Collections.sort(sortedData, new Comparator<Tuple2<Integer, Long>>() {
								@Override
								public int compare(Tuple2<Integer, Long> o1, Tuple2<Integer, Long> o2) {
									return o1.f0.compareTo(o2.f0);
								}
							});
							return sortedData;
						}
					});

			// compute the offset for each partition
			for (int i = 0; i < getRuntimeContext().getIndexOfThisSubtask(); i++) {
				start += offsets.get(i).f1;
			}
		}

		@Override
		public void mapPartition(Iterable<T> values, Collector<Tuple2<Long, T>> out) throws Exception {
			for (T value: values) {
				out.collect(new Tuple2<>(start++, value));
			}
		}
	}).withBroadcastSet(elementCount, "counts");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported in rich async functions.");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(String name, BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables can only be used in DataSet programs");
}
 

@Override
public <T, C> C getBroadcastVariableWithInitializer(
		final String name,
		final BroadcastVariableInitializer<T, C> initializer) {
	throw new UnsupportedOperationException("Broadcast variables are not supported.");
}