Java Code Examples for org.apache.flink.testutils.TestFileUtils#createTempFileInDirectory()

The following examples show how to use org.apache.flink.testutils.TestFileUtils#createTempFileInDirectory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 2

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 3

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 4

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 5

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 6

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

6 votes

/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 7

Source File: EnumerateNestedFilesTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 8

Source File: EnumerateNestedFilesTest.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 9

Source File: EnumerateNestedFilesTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 10

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 11

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();

		File nestedNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir);
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);
		
		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 12

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests if the recursion is invoked correctly in nested directories.
 */
@Test
public void testOnlyLevel2NestedDirectories() {
	try {
		String rootDir = TestFileUtils.randomFileName();
		String nestedDir = TestFileUtils.randomFileName();
		String firstNestedNestedDir = TestFileUtils.randomFileName();
		String secondNestedNestedDir = TestFileUtils.randomFileName();

		File testDir = tempFolder.newFolder(rootDir);
		tempFolder.newFolder(rootDir, nestedDir);
		File nestedNestedDir1 = tempFolder.newFolder(rootDir, nestedDir, firstNestedNestedDir);
		File nestedNestedDir2 = tempFolder.newFolder(rootDir, nestedDir, secondNestedNestedDir);

		// create files in second level
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(testDir.getAbsolutePath()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 13

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesWithFilteredFilesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();
		String secondLevelFilterDir = "_"+TestFileUtils.randomFileName();
		String thirdLevelFilterDir = "_"+TestFileUtils.randomFileName();

		File nestedNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir, thirdLevelFilterDir);
		File nestedNestedDir = nestedNestedDirFiltered.getParentFile();
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();
		File insideNestedDirFiltered = tempFolder.newFolder(firstLevelDir, secondLevelFilterDir);
		File filteredFile = new File(nestedDir, "_IWillBeFiltered");
		filteredFile.createNewFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");
		// create files which are filtered
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDirFiltered.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDirFiltered.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 14

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests if the recursion is invoked correctly in nested directories.
 */
@Test
public void testOnlyLevel2NestedDirectories() {
	try {
		String rootDir = TestFileUtils.randomFileName();
		String nestedDir = TestFileUtils.randomFileName();
		String firstNestedNestedDir = TestFileUtils.randomFileName();
		String secondNestedNestedDir = TestFileUtils.randomFileName();

		File testDir = tempFolder.newFolder(rootDir);
		tempFolder.newFolder(rootDir, nestedDir);
		File nestedNestedDir1 = tempFolder.newFolder(rootDir, nestedDir, firstNestedNestedDir);
		File nestedNestedDir2 = tempFolder.newFolder(rootDir, nestedDir, secondNestedNestedDir);

		// create files in second level
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir1.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir2.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(testDir.getAbsolutePath()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 15

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

5 votes

/**
 * Test with two nested directories and recursive.file.enumeration = true
 */
@Test
public void testTwoNestedDirectoriesTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String thirdLevelDir = TestFileUtils.randomFileName();

		File nestedNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir, thirdLevelDir);
		File insideNestedDir = nestedNestedDir.getParentFile();
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level, two files in the second level and one in the third level
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");
		TestFileUtils.createTempFileInDirectory(nestedNestedDir.getAbsolutePath(), "bravas");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);
		
		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(4, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 16

Source File: EnumerateNestedFilesTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testGetStatisticsMultipleNestedFiles() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long SIZE4 = 71;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3 + SIZE4;

		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir2 = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File insideNestedDir2 = tempFolder.newFolder(firstLevelDir, secondLevelDir2);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), SIZE1);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE2);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE3);
		TestFileUtils.createTempFileInDirectory(insideNestedDir2.getAbsolutePath(), SIZE4);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());

		/* Now invalidate the cache and check again */
		Thread.sleep(1000); // accuracy of file modification times is rather low
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), 42L);

		BaseStatistics stats2 = format.getStatistics(stats);
		Assert.assertNotEquals(stats2, stats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL + 42L, stats2.getTotalInputSize());

	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 17

Source File: FileInputFormatTest.java From flink with Apache License 2.0

4 votes

@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 18

Source File: EnumerateNestedFilesTest.java From flink with Apache License 2.0

4 votes

@Test
public void testGetStatisticsMultipleNestedFiles() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long SIZE4 = 71;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3 + SIZE4;

		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir2 = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File insideNestedDir2 = tempFolder.newFolder(firstLevelDir, secondLevelDir2);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), SIZE1);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE2);
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE3);
		TestFileUtils.createTempFileInDirectory(insideNestedDir2.getAbsolutePath(), SIZE4);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());

		/* Now invalidate the cache and check again */
		Thread.sleep(1000); // accuracy of file modification times is rather low
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), 42L);

		BaseStatistics stats2 = format.getStatistics(stats);
		Assert.assertNotEquals(stats2, stats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL + 42L, stats2.getTotalInputSize());

	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 19

Source File: FileInputFormatTest.java From flink with Apache License 2.0

4 votes

@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}

Example 20

Source File: FileInputFormatTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testGetStatisticsMultipleFilesWithCachedVersion() {
	try {
		FileSystem fs = FileSystem.getLocalFileSystem();

		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		final long FAKE_SIZE = 10065;
		
		File tempDirFile = temporaryFolder.newFolder();
		String tempDir = tempDirFile.getAbsolutePath();
		String f1 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE1);
		long modTime1 = fs.getFileStatus(new Path(f1)).getModificationTime();
		String f2 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE2);
		long modTime2 = fs.getFileStatus(new Path(f2)).getModificationTime();
		String f3 = TestFileUtils.createTempFileInDirectory(tempDir, SIZE3);
		long modTime3 = fs.getFileStatus(new Path(f3)).getModificationTime();
		
		DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
		
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics newStats = format.getStatistics(stats);
		Assert.assertTrue("Statistics object was changed", newStats == stats);

		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics fakeStats = new FileBaseStatistics(stats.getLastModificationTime(), FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics latest = format.getStatistics(fakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", FAKE_SIZE, latest.getTotalInputSize());
		
		// insert fake stats with the correct modification time. the call should return the fake stats
		format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		FileBaseStatistics outDatedFakeStats = new FileBaseStatistics(Math.min(Math.min(modTime1, modTime2), modTime3) - 1, FAKE_SIZE, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
		BaseStatistics reGathered = format.getStatistics(outDatedFakeStats);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, reGathered.getTotalInputSize());
		
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}