Java Code Examples for org.apache.flink.testutils.TestFileUtils

The following examples show how to use org.apache.flink.testutils.TestFileUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
 
Example 4
Source Project: Flink-CEPplus   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
 
Example 5
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 6
@Test
public void testDifferentDelimiter() {
	try {
		final String DELIMITER = "12345678-";
		String testData = TEST_DATA1.replace("\n", DELIMITER);
		
		final String tempFile = TestFileUtils.createTempFile(testData);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.setDelimiter(DELIMITER);
		format.configure(conf);
		
		BaseStatistics stats = format.getStatistics(null);
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES;
		
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 7
@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 8
Source Project: Flink-CEPplus   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 10
Source Project: Flink-CEPplus   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 12
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 13
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 14
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
 
Example 15
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
 
Example 16
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 17
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDifferentDelimiter() {
	try {
		final String DELIMITER = "12345678-";
		String testData = TEST_DATA1.replace("\n", DELIMITER);
		
		final String tempFile = TestFileUtils.createTempFile(testData);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.setDelimiter(DELIMITER);
		format.configure(conf);
		
		BaseStatistics stats = format.getStatistics(null);
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES;
		
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 18
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 19
Source Project: flink   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 20
Source Project: flink   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 21
Source Project: flink   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 22
Source Project: flink   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 23
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 24
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 25
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
 
Example 26
Source Project: flink   Source File: FileInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
 
Example 27
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 28
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDifferentDelimiter() {
	try {
		final String DELIMITER = "12345678-";
		String testData = TEST_DATA1.replace("\n", DELIMITER);
		
		final String tempFile = TestFileUtils.createTempFile(testData);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.setDelimiter(DELIMITER);
		format.configure(conf);
		
		BaseStatistics stats = format.getStatistics(null);
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES;
		
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 29
Source Project: flink   Source File: DelimitedInputFormatSamplingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 30
Source Project: flink   Source File: EnumerateNestedFilesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}