org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit Java Examples

The following examples show how to use org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HadoopInputFormatBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example #2
Source File: HadoopInputFormatBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #3
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example #4
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #5
Source File: TapInputFormat.java    From cascading-flink with Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {

	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example #6
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 
Example #7
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 
Example #8
Source File: HadoopInputFormatBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example #9
Source File: HadoopInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 
Example #10
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example #11
Source File: HadoopInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 
Example #12
Source File: TapInputFormat.java    From cascading-flink with Apache License 2.0 4 votes vote down vote up
@Override
public void open(HadoopInputSplit split) throws IOException {

	this.jobConf = split.getJobConf();
	this.flowProcess = new FlinkFlowProcess(this.jobConf, this.getRuntimeContext(), flowNode.getID());

	processBeginTime = System.currentTimeMillis();
	flowProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );

	try {

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for TapInputFormat may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof Tap)) {
			throw new RuntimeException("Source of TapInputFormat must be a Tap");
		}
		Tap source = (Tap)sourceElement;

		streamGraph = new SourceStreamGraph( flowProcess, flowNode, source );

		sourceStage = this.streamGraph.getSourceStage();
		sinkStage = this.streamGraph.getSinkStage();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}

	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during TapInputFormat configuration", throwable );
	}

	RecordReader<?, ?> recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());

	if (recordReader instanceof Configurable) {
		((Configurable) recordReader).setConf(jobConf);
	}
	else if (recordReader instanceof JobConfigurable) {
		((JobConfigurable) recordReader).configure(jobConf);
	}

	try {
		this.sourceStage.setRecordReader(recordReader);
	} catch(Throwable t) {
		if(t instanceof IOException) {
			throw (IOException)t;
		}
		else {
			throw new RuntimeException(t);
		}
	}

}
 
Example #13
Source File: TapInputFormat.java    From cascading-flink with Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example #14
Source File: HadoopInputFormatBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 
Example #15
Source File: HadoopInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}