Java Code Examples for org.apache.flink.api.common.io.RichOutputFormat

The following examples show how to use org.apache.flink.api.common.io.RichOutputFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: alibaba-flink-connectors   Author: alibaba   File: OutputFormatSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration config) throws IOException {
	if (RichOutputFormat.class.isAssignableFrom(outputFormat.getClass())) {
		((RichOutputFormat) outputFormat).setRuntimeContext(getRuntimeContext());
	}
	outputFormat.configure(config);
	outputFormat.open(
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
	if (outputFormat instanceof HasRetryTimeout) {
		retryTimeout = ((HasRetryTimeout) outputFormat).getRetryTimeout();
	}
	LOG.info(
		"Initialized OutputFormatSinkFunction of {}/{} task.",
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
}
 
Example #2
Source Project: alibaba-flink-connectors   Author: alibaba   File: TupleOutputFormatSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration config) throws IOException {
	if (RichOutputFormat.class.isAssignableFrom(outputFormat.getClass())) {
		((RichOutputFormat) outputFormat).setRuntimeContext(getRuntimeContext());
	}
	outputFormat.configure(config);
	outputFormat.open(
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
	if (outputFormat instanceof HasRetryTimeout) {
		retryTimeout = ((HasRetryTimeout) outputFormat).getRetryTimeout();
	}
	LOG.info(
		"Initialized OutputFormatSinkFunction of {}/{} task.",
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: OutputFormatSinkFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: OutputFormatSinkFunctionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: OutputFormatSinkFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: OutputFormatSinkFunctionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
 
Example #9
Source Project: Alink   Author: alibaba   File: JdbcDB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    TypeInformation[] types = schema.getFieldTypes();
    String[] colNames = schema.getFieldNames();
    int[] parameterTypes = new int[types.length];
    for (int i = 0; i < types.length; ++i) {
        parameterTypes[i] = JdbcTypeConverter.getIntegerSqlType(types[i]);
    }
    StringBuilder sbd = new StringBuilder();
    sbd.append("INSERT INTO ").append(tableName).append(" (").append(colNames[0]);
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append(colNames[i]);
    }
    sbd.append(") VALUES (?");
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append("?");
    }
    sbd.append(")");

    return JDBCOutputFormat.buildJDBCOutputFormat()
            .setUsername(userName)
            .setPassword(password)
            .setDBUrl(getDbUrl())
            .setQuery(sbd.toString())
            .setDrivername(getDriverName())
            .setBatchInterval(5000)
            .setSqlTypes(parameterTypes)
            .finish();
}
 
Example #10
Source Project: flink   Author: apache   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #11
Source Project: flink   Author: apache   File: OutputFormatSinkFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
 
Example #12
Source Project: flink   Author: apache   File: OutputFormatSinkFunctionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: GenericDataSinkBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: GenericDataSinkBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example #15
Source Project: Alink   Author: alibaba   File: FakeDBBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    return null;
}
 
Example #16
Source Project: Alink   Author: alibaba   File: HiveDB.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    throw new UnsupportedOperationException("not supported.");
}
 
Example #17
Source Project: flink   Author: apache   File: GenericDataSinkBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example #18
Source Project: Alink   Author: alibaba   File: BaseDB.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Create Flink OutputFormat from a table in db.
 *
 * @param tableName table name
 * @param schema    format schema.
 * @return Flink OutputFormat
 */
public abstract RichOutputFormat createFormat(String tableName, TableSchema schema);