Java Code Examples for org.apache.flink.testutils.TestFileUtils#createTempFileInDirectory()

The following examples show how to use org.apache.flink.testutils.TestFileUtils#createTempFileInDirectory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 2
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 3
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 4
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 5
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 6
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 7
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 8
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 9
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 10
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 11
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();

		File nestedNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir);
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);
		
		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 12
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests if the recursion is invoked correctly in nested directories.
 */
@Test
public void testOnlyLevel2NestedDirectories() {
	try {
		String rootDir = TestFileUtils.randomFileName();
		String nestedDir = TestFileUtils.randomFileName();
		String firstNestedNestedDir = TestFileUtils.randomFileName();
		String secondNestedNestedDir = TestFileUtils.randomFileName();

		File testDir = tempFolder.newFolder(rootDir);
		tempFolder.newFolder(rootDir, nestedDir);
		File nestedNestedDir1 = tempFolder.newFolder(rootDir, nestedDir, firstNestedNestedDir);
		File nestedNestedDir2 = tempFolder.newFolder(rootDir, nestedDir, secondNestedNestedDir);

		// create files in second level
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(testDir.getAbsolutePath()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 13
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 14
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests if the recursion is invoked correctly in nested directories.
 */
@Test
public void testOnlyLevel2NestedDirectories() {
	try {
		String rootDir = TestFileUtils.randomFileName();
		String nestedDir = TestFileUtils.randomFileName();
		String firstNestedNestedDir = TestFileUtils.randomFileName();
		String secondNestedNestedDir = TestFileUtils.randomFileName();

		File testDir = tempFolder.newFolder(rootDir);
		tempFolder.newFolder(rootDir, nestedDir);
		File nestedNestedDir1 = tempFolder.newFolder(rootDir, nestedDir, firstNestedNestedDir);
		File nestedNestedDir2 = tempFolder.newFolder(rootDir, nestedDir, secondNestedNestedDir);

		// create files in second level
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(testDir.getAbsolutePath()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 15
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();

		File nestedNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir);
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);
		
		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 16
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetStatisticsMultipleNestedFiles() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long SIZE4 = 71;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3 + SIZE4;

		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir2 = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File insideNestedDir2 = tempFolder.newFolder(firstLevelDir, secondLevelDir2);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), SIZE1);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE2);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE3);
		TestFileUtils.createTempFileInDirectory(insideNestedDir2.getAbsolutePath(), SIZE4);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());

		/* Now invalidate the cache and check again */
		Thread.sleep(1000); // accuracy of file modification times is rather low
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), 42L);

		BaseStatistics stats2 = format.getStatistics(stats);
		Assert.assertNotEquals(stats2, stats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL + 42L, stats2.getTotalInputSize());

	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 17
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 18
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetStatisticsMultipleNestedFiles() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long SIZE4 = 71;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3 + SIZE4;

		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir2 = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File insideNestedDir2 = tempFolder.newFolder(firstLevelDir, secondLevelDir2);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), SIZE1);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE2);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE3);
		TestFileUtils.createTempFileInDirectory(insideNestedDir2.getAbsolutePath(), SIZE4);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());

		/* Now invalidate the cache and check again */
		Thread.sleep(1000); // accuracy of file modification times is rather low
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), 42L);

		BaseStatistics stats2 = format.getStatistics(stats);
		Assert.assertNotEquals(stats2, stats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL + 42L, stats2.getTotalInputSize());

	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 19
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
 
Example 20
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}