org.apache.flink.types.Record Java Examples

@Test
public void testGetStatisticsMultiplePaths() throws IOException {
	final int blockInfoSize = new BlockInfo().getInfoSize();
	final int blockSize = blockInfoSize + 8;
	final int numBlocks1 = 3;
	final int numBlocks2 = 5;
	
	final File tempFile = createBinaryInputFile("binary_input_format_test", blockSize, numBlocks1);
	final File tempFile2 = createBinaryInputFile("binary_input_format_test_2", blockSize, numBlocks2);
	
	final BinaryInputFormat<Record> inputFormat = new MyBinaryInputFormat();
	inputFormat.setFilePaths(tempFile.toURI().toString(), tempFile2.toURI().toString());
	inputFormat.setBlockSize(blockSize);

	BaseStatistics stats = inputFormat.getStatistics(null);
	Assert.assertEquals("The file size statistics is wrong", blockSize * (numBlocks1 + numBlocks2), stats.getTotalInputSize());
}
 

@Override
public Iterator<Record> iterator() {
	
	return new Iterator<Record>() {

		@Override
		public boolean hasNext() {
			return false;
		}

		@Override
		public Record next() {
			return null;
		}

		@Override
		public void remove() {
			throw new UnsupportedOperationException();
		}
		
	};
}
 

@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 

@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 

@Override
public TypePairComparator<Record, Record> createComparator12(
		TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos1, pos2, types1);
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.value);

		sum += this.value.getValue();
	}
	element.getField(0, this.key);
	this.value.setValue(sum - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);

		sum += this.combineValue.getValue();
	}

	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

public static void prepareInputFile(MutableObjectIterator<Record> inIt, File inputFile, boolean insertInvalidData)
throws IOException {

	try (BufferedWriter bw = new BufferedWriter(new FileWriter(inputFile))) {
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		Record rec = new Record();
		while ((rec = inIt.next(rec)) != null) {
			IntValue key = rec.getField(0, IntValue.class);
			IntValue value = rec.getField(1, IntValue.class);

			bw.write(key.getValue() + "_" + value.getValue() + "\n");
		}
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		bw.flush();
	}
}
 

@Override
public Record readRecord(Record target, byte[] record, int offset, int numBytes) {
	
	String line = new String(record, offset, numBytes, ConfigConstants.DEFAULT_CHARSET);
	
	try {
		this.key.setValue(Integer.parseInt(line.substring(0,line.indexOf("_"))));
		this.value.setValue(Integer.parseInt(line.substring(line.indexOf("_")+1,line.length())));
	}
	catch(RuntimeException re) {
		return null;
	}
	
	target.setField(0, this.key);
	target.setField(1, this.value);
	return target;
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;
	
	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);
		
		sum += this.combineValue.getValue();
	}
	
	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);

		sum += this.combineValue.getValue();
	}

	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

@Override
public boolean equalToReference(Record candidate) {
	try {
		return this.key == candidate.getField(0, IntValue.class).getValue();
	} catch (NullPointerException npex) {
		throw new NullKeyFieldException();
	}
}
 

@Override
protected BinaryOutputFormat<Record> createOutputFormat(String path, Configuration configuration)
		throws IOException {
	final SerializedOutputFormat<Record> outputFormat = new SerializedOutputFormat<Record>();
	outputFormat.setOutputFilePath(new Path(path));
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	configuration = configuration == null ? new Configuration() : configuration;
	outputFormat.configure(configuration);
	outputFormat.open(0, 1);
	return outputFormat;
}
 

@Test
public void testExternalHash1MatchTask() {
	final int keyCnt1 = 32768;
	final int valCnt1 = 8;
	
	final int keyCnt2 = 65536;
	final int valCnt2 = 8;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.output);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 

@Before
public void startup() {
	// set up IO and memory manager
	this.memman = new MemoryManager(MEMORY_CAPACITY, 1);
	
	// create test objects
	this.objects = new ArrayList<Record>(20000);
	for (int i = 0; i < NUM_VALUES; ++i) {
		this.objects.add(new Record(new IntValue(i)));
	}
	
	// create the reader
	this.reader = objects.iterator();
}
 

@Override
protected ValueTypeInfo<?>[] getTestData() {
	return new ValueTypeInfo<?>[] {
		new ValueTypeInfo<>(TestClass.class),
		new ValueTypeInfo<>(AlternativeClass.class),
		new ValueTypeInfo<>(Record.class),
	};
}
 

@Test
public void testCreateInputSplitsWithMulitpleFiles() throws IOException {
	final int blockInfoSize = new BlockInfo().getInfoSize();
	final int blockSize = blockInfoSize + 8;
	final int numBlocks1 = 3;
	final int numBlocks2 = 5;
	
	final File tempFile1 = createBinaryInputFile("binary_input_format_test", blockSize, numBlocks1);
	final File tempFile2 = createBinaryInputFile("binary_input_format_test_2", blockSize, numBlocks2);
	final String pathFile1 = tempFile1.toURI().toString(); 
	final String pathFile2 = tempFile2.toURI().toString(); 
	
	final BinaryInputFormat<Record> inputFormat = new MyBinaryInputFormat();
	inputFormat.setFilePaths(pathFile1, pathFile2);
	inputFormat.setBlockSize(blockSize);
	
	final int numBlocksTotal = numBlocks1 + numBlocks2;
	FileInputSplit[] inputSplits = inputFormat.createInputSplits(numBlocksTotal);
	
	int numSplitsFile1 = 0;
	int numSplitsFile2 = 0;
	
	Assert.assertEquals("Returns requested numbers of splits.", numBlocksTotal, inputSplits.length);
	for (int i = 0; i < inputSplits.length; i++) {
		Assert.assertEquals(String.format("%d. split has block size length.", i), blockSize, inputSplits[i].getLength());
		if (inputSplits[i].getPath().toString().equals(pathFile1)) {
			numSplitsFile1++;
		} else if (inputSplits[i].getPath().toString().equals(pathFile2)) {
			numSplitsFile2++;
		} else {
			Assert.fail("Split does not belong to any input file.");
		}
	}
	Assert.assertEquals(numBlocks1, numSplitsFile1);
	Assert.assertEquals(numBlocks2, numSplitsFile2);
}
 

@Override
public void writeRecord(Record rec) throws IOException {
	if (++this.cnt >= 10) {
		throw new RuntimeException("Expected Test Exception");
	}
	super.writeRecord(rec);
}
 

@Test
public void testSortSecondCoGroupTask() {
	int keyCnt1 = 200;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 4;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 

@Test
public void testMergeMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 

@Test
public void testSortBoth1CoGroupTask() {
	int keyCnt1 = 100;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 1;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	for (@SuppressWarnings("unused") Record r : records) {
		try {
			Thread.sleep(100);
		} catch (InterruptedException e) {}
	}
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int cnt = 0;
	
	for (Record next : records) {
		element = next;
		cnt++;
	}
	element.getField(0, this.key);
	this.value.setValue(cnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 

@Test
public void testFailingDataSourceTask() throws IOException {
	int keyCnt = 20;
	int valCnt = 10;
	
	this.outList = new NirvanaOutputList();
	File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
	InputFilePreparator.prepareInputFile(new UniformRecordGenerator(keyCnt, valCnt, false),
		tempTestFile, false);

	super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	super.addOutput(this.outList);
	
	DataSourceTask<Record> testTask = new DataSourceTask<>(this.mockEnv);

	super.registerFileInputTask(testTask, MockFailingInputFormat.class, tempTestFile.toURI().toString(), "\n");
	
	boolean stubFailed = false;

	try {
		testTask.invoke();
	} catch (Exception e) {
		stubFailed = true;
	}
	Assert.assertTrue("Function exception was not forwarded.", stubFailed);
	
	// assert that temp file was created
	Assert.assertTrue("Temp output file does not exist",tempTestFile.exists());
	
}
 

@Test
public void testSortBoth4MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 

public IteratorWrappingTestSingleInputGate<Record> addInput(MutableObjectIterator<Record> inputIterator) {
	try {
		final IteratorWrappingTestSingleInputGate<Record> reader = new IteratorWrappingTestSingleInputGate<Record>(bufferSize, Record.class, inputIterator);

		inputs.add(reader.getInputGate());

		return reader;
	}
	catch (Throwable t) {
		throw new RuntimeException("Error setting up mock readers: " + t.getMessage(), t);
	}
}
 

@Test
public void testHash1MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 10;
	int valCnt2 = 2;
			
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 

@Before
public void startup() {
	// set up IO and memory manager
	this.memman = new MemoryManager(MEMORY_CAPACITY, 1);
	
	// create test objects
	this.objects = new ArrayList<Record>(20000);
	for (int i = 0; i < NUM_VALUES; ++i) {
		this.objects.add(new Record(new IntValue(i)));
	}
	
	// create the reader
	this.reader = new MutableObjectIteratorWrapper(this.objects.iterator());
}
 

public IteratorWrappingTestSingleInputGate<Record> addInput(MutableObjectIterator<Record> input, int groupId, boolean read) {
	final IteratorWrappingTestSingleInputGate<Record> reader = this.mockEnv.addInput(input);
	TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
	conf.addInputToGroup(groupId);
	conf.setInputSerializer(RecordSerializerFactory.get(), groupId);

	if (read) {
		reader.notifyNonEmpty();
	}

	return reader;
}