Java Code Examples for org.apache.flink.api.common.cache.DistributedCache#DistributedCacheEntry

The following examples show how to use org.apache.flink.api.common.cache.DistributedCache#DistributedCacheEntry . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FileCacheReadsFromBlobTest.java From flink with Apache License 2.0

6 votes

@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}

Example 2

Source File: FileCacheReadsFromBlobTest.java From flink with Apache License 2.0

6 votes

@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}

Example 3

Source File: FileCacheDirectoriesTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}

Example 4

Source File: PlanGenerator.java From flink with Apache License 2.0

5 votes

public PlanGenerator(
		List<DataSink<?>> sinks,
		ExecutionConfig config,
		int defaultParallelism,
		List<Tuple2<String, DistributedCache.DistributedCacheEntry>> cacheFile,
		String jobName) {
	this.sinks = checkNotNull(sinks);
	this.config = checkNotNull(config);
	this.cacheFile = checkNotNull(cacheFile);
	this.jobName = checkNotNull(jobName);
	this.defaultParallelism = defaultParallelism;
}

Example 5

Source File: FileCacheDirectoriesTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDirectoryCleanUp() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID1 = new ExecutionAttemptID();
	ExecutionAttemptID attemptID2 = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID1);
	fileCache.createTmpFile(fileName, entry, jobID, attemptID2);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID1);
	// still should be available
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID2);
	// still should be available, file will be deleted after cleanupInterval
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	// after a while, the file should disappear
	assertEquals(CLEANUP_INTERVAL, executorService.lastDelayMillis);
	executorService.lastDeleteProcess.run();

	assertFalse(fs.exists(dstPath));
	assertFalse(fs.exists(cacheFile));
}

Example 6

Source File: FileCacheDirectoriesTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		"test_file",
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);

	testDirectoryDownloaded(entry);
}

Example 7

Source File: JobGraph.java From flink with Apache License 2.0

5 votes

public void writeUserArtifactEntriesToConfiguration() {
	for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact : userArtifacts.entrySet()) {
		DistributedCache.writeFileInfoToConfig(
			userArtifact.getKey(),
			userArtifact.getValue(),
			jobConfiguration
		);
	}
}

Example 8

Source File: JobGraphGeneratorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

private static void assertState(DistributedCache.DistributedCacheEntry entry, boolean isExecutable, boolean isZipped) throws IOException {
	assertNotNull(entry);
	assertEquals(isExecutable, entry.isExecutable);
	assertEquals(isZipped, entry.isZipped);
	org.apache.flink.core.fs.Path filePath = new org.apache.flink.core.fs.Path(entry.filePath);
	assertTrue(filePath.getFileSystem().exists(filePath));
	assertFalse(filePath.getFileSystem().getFileStatus(filePath).isDir());
}

Example 9

Source File: JobGraph.java From flink with Apache License 2.0

5 votes

/**
 * Adds the path of a custom file required to run the job on a task manager.
 *
 * @param name a name under which this artifact will be accessible through {@link DistributedCache}
 * @param file path of a custom file required to run the job on a task manager
 */
public void addUserArtifact(String name, DistributedCache.DistributedCacheEntry file) {
	if (file == null) {
		throw new IllegalArgumentException();
	}

	userArtifacts.putIfAbsent(name, file);
}

Example 10

Source File: FileCacheDirectoriesTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDirectoryCleanUp() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID1 = new ExecutionAttemptID();
	ExecutionAttemptID attemptID2 = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID1);
	fileCache.createTmpFile(fileName, entry, jobID, attemptID2);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID1);
	// still should be available
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID2);
	// still should be available, file will be deleted after cleanupInterval
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	// after a while, the file should disappear
	assertEquals(CLEANUP_INTERVAL, executorService.lastDelayMillis);
	executorService.lastDeleteProcess.run();

	assertFalse(fs.exists(dstPath));
	assertFalse(fs.exists(cacheFile));
}

Example 11

Source File: JobGraphGeneratorTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphGenerator.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}

Example 12

Source File: JobGraphGeneratorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphUtils.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}

Example 13

Source File: StreamGraph.java From flink with Apache License 2.0

4 votes

public Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> getUserArtifacts() {
	return userArtifacts;
}

Example 14

Source File: CollectionExecutor.java From flink with Apache License 2.0

4 votes

private void initCache(Set<Map.Entry<String, DistributedCache.DistributedCacheEntry>> files){
	for(Map.Entry<String, DistributedCache.DistributedCacheEntry> file: files){
		Future<Path> doNothing = new CompletedFuture(new Path(file.getValue().filePath));
		cachedFiles.put(file.getKey(), doNothing);
	}
}

Example 15

Source File: StreamGraph.java From flink with Apache License 2.0

4 votes

public void setUserArtifacts(Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> userArtifacts) {
	this.userArtifacts = userArtifacts;
}

Example 16

Source File: JobGraphGeneratorTest.java From flink with Apache License 2.0

4 votes

@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphGenerator.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}

Example 17

Source File: CollectionExecutor.java From Flink-CEPplus with Apache License 2.0

4 votes

private void initCache(Set<Map.Entry<String, DistributedCache.DistributedCacheEntry>> files){
	for(Map.Entry<String, DistributedCache.DistributedCacheEntry> file: files){
		Future<Path> doNothing = new CompletedFuture(new Path(file.getValue().filePath));
		cachedFiles.put(file.getKey(), doNothing);
	}
}

Example 18

Source File: JobGraph.java From flink with Apache License 2.0

2 votes

/**
 * Gets the list of assigned user jar paths.
 *
 * @return The list of assigned user jar paths
 */
public Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts() {
	return userArtifacts;
}

Example 19

Source File: JobGraph.java From Flink-CEPplus with Apache License 2.0

2 votes

/**
 * Gets the list of assigned user jar paths.
 *
 * @return The list of assigned user jar paths
 */
public Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts() {
	return userArtifacts;
}

Example 20

Source File: PlanGenerator.java From flink with Apache License 2.0

2 votes

/**
 * Registers all files that were registered at this execution environment's cache registry of the
 * given plan's cache registry.
 *
 * @param p The plan to register files at.
 * @throws IOException Thrown if checks for existence and sanity fail.
 */
private void registerCachedFilesWithPlan(Plan p) throws IOException {
	for (Tuple2<String, DistributedCache.DistributedCacheEntry> entry : cacheFile) {
		p.registerCachedFile(entry.f0, entry.f1);
	}
}