org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopDummyReporter Java Examples

The following examples show how to use org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopDummyReporter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HadoopReduceCombineFunction.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);
	this.combiner.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<>(keySerializer);
	this.combineCollector = new HadoopOutputCollector<>();
	this.reduceCollector = new HadoopOutputCollector<>();
}
 
Example #2
Source File: HadoopReduceFunction.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(keySerializer);
}
 
Example #3
Source File: HadoopMapFunction.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.mapper.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
}
 
Example #4
Source File: HadoopInputFormatBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #5
Source File: HadoopOutputFormatBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * commit the task by moving the output file out from the temporary directory.
 * @throws java.io.IOException
 */
@Override
public void close() throws IOException {

	// enforce sequential close() calls
	synchronized (CLOSE_MUTEX) {
		this.recordWriter.close(new HadoopDummyReporter());

		if (this.outputCommitter.needsTaskCommit(this.context)) {
			this.outputCommitter.commitTask(this.context);
		}
	}
}
 
Example #6
Source File: HadoopReduceCombineFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);
	this.combiner.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<>(keySerializer);
	this.combineCollector = new HadoopOutputCollector<>();
	this.reduceCollector = new HadoopOutputCollector<>();
}
 
Example #7
Source File: HadoopReduceFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(keySerializer);
}
 
Example #8
Source File: HadoopMapFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.mapper.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
}
 
Example #9
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #10
Source File: HadoopOutputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * commit the task by moving the output file out from the temporary directory.
 * @throws java.io.IOException
 */
@Override
public void close() throws IOException {

	// enforce sequential close() calls
	synchronized (CLOSE_MUTEX) {
		this.recordWriter.close(new HadoopDummyReporter());

		if (this.outputCommitter.needsTaskCommit(this.context)) {
			this.outputCommitter.commitTask(this.context);
		}
	}
}
 
Example #11
Source File: HadoopReduceCombineFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);
	this.combiner.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<>(keySerializer);
	this.combineCollector = new HadoopOutputCollector<>();
	this.reduceCollector = new HadoopOutputCollector<>();
}
 
Example #12
Source File: HadoopReduceFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(keySerializer);
}
 
Example #13
Source File: HadoopMapFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.mapper.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
}
 
Example #14
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #15
Source File: HadoopOutputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * commit the task by moving the output file out from the temporary directory.
 * @throws java.io.IOException
 */
@Override
public void close() throws IOException {

	// enforce sequential close() calls
	synchronized (CLOSE_MUTEX) {
		this.recordWriter.close(new HadoopDummyReporter());

		if (this.outputCommitter.needsTaskCommit(this.context)) {
			this.outputCommitter.commitTask(this.context);
		}
	}
}
 
Example #16
Source File: TapInputFormat.java    From cascading-flink with Apache License 2.0 4 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	this.jobConf = split.getJobConf();
	this.flowProcess = new FlinkFlowProcess(this.jobConf, this.getRuntimeContext(), flowNode.getID());

	processBeginTime = System.currentTimeMillis();
	flowProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );

	try {

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for TapInputFormat may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof Tap)) {
			throw new RuntimeException("Source of TapInputFormat must be a Tap");
		}
		Tap source = (Tap)sourceElement;

		streamGraph = new SourceStreamGraph( flowProcess, flowNode, source );

		sourceStage = this.streamGraph.getSourceStage();
		sinkStage = this.streamGraph.getSinkStage();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}

	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during TapInputFormat configuration", throwable );
	}

	RecordReader<?, ?> recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());

	if (recordReader instanceof Configurable) {
		((Configurable) recordReader).setConf(jobConf);
	}
	else if (recordReader instanceof JobConfigurable) {
		((JobConfigurable) recordReader).configure(jobConf);
	}

	try {
		this.sourceStage.setRecordReader(recordReader);
	} catch(Throwable t) {
		if(t instanceof IOException) {
			throw (IOException)t;
		}
		else {
			throw new RuntimeException(t);
		}
	}

}