Java Code Examples for org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit

The following examples show how to use org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example 2
Source Project: Flink-CEPplus   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example 3
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example 4
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example 5
Source Project: cascading-flink   Source File: TapInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {

	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example 6
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example 7
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example 8
Source Project: Flink-CEPplus   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: HadoopInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 
Example 10
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example 11
Source Project: flink   Source File: HadoopInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 
Example 12
Source Project: cascading-flink   Source File: TapInputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	this.jobConf = split.getJobConf();
	this.flowProcess = new FlinkFlowProcess(this.jobConf, this.getRuntimeContext(), flowNode.getID());

	processBeginTime = System.currentTimeMillis();
	flowProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );

	try {

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for TapInputFormat may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof Tap)) {
			throw new RuntimeException("Source of TapInputFormat must be a Tap");
		}
		Tap source = (Tap)sourceElement;

		streamGraph = new SourceStreamGraph( flowProcess, flowNode, source );

		sourceStage = this.streamGraph.getSourceStage();
		sinkStage = this.streamGraph.getSinkStage();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}

	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during TapInputFormat configuration", throwable );
	}

	RecordReader<?, ?> recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());

	if (recordReader instanceof Configurable) {
		((Configurable) recordReader).setConf(jobConf);
	}
	else if (recordReader instanceof JobConfigurable) {
		((JobConfigurable) recordReader).configure(jobConf);
	}

	try {
		this.sourceStage.setRecordReader(recordReader);
	} catch(Throwable t) {
		if(t instanceof IOException) {
			throw (IOException)t;
		}
		else {
			throw new RuntimeException(t);
		}
	}

}
 
Example 13
Source Project: cascading-flink   Source File: TapInputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example 14
Source Project: flink   Source File: HadoopInputFormatBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example 15
Source Project: flink   Source File: HadoopInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}