org.apache.flink.runtime.blob.TransientBlobKey Java Examples

The following examples show how to use org.apache.flink.runtime.blob.TransientBlobKey. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

6 votes

private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}

Example #2

Source File: TaskExecutor.java From flink with Apache License 2.0

6 votes

private CompletableFuture<TransientBlobKey> requestFileUploadByFilePath(String filePath, String fileTag) {
	log.debug("Received file upload request for file {}", fileTag);
	if (!StringUtils.isNullOrWhitespaceOnly(filePath)) {
		return CompletableFuture.supplyAsync(() -> {
			final File file = new File(filePath);
			if (file.exists()) {
				try {
					return putTransientBlobStream(new FileInputStream(file), fileTag).get();
				} catch (Exception e) {
					log.debug("Could not upload file {}.", fileTag, e);
					throw new CompletionException(new FlinkException("Could not upload file " + fileTag + '.', e));
				}
			} else {
				log.debug("The file {} does not exist on the TaskExecutor {}.", fileTag, getResourceID());
				throw new CompletionException(new FlinkException("The file " + fileTag + " does not exist on the TaskExecutor."));
			}
		}, ioExecutor);
	} else {
		log.debug("The file {} is unavailable on the TaskExecutor {}.", fileTag, getResourceID());
		return FutureUtils.completedExceptionally(new FlinkException("The file " + fileTag + " is not available on the TaskExecutor."));
	}
}

Example #3

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

6 votes

/**
 * Tests that the {@link AbstractTaskManagerFileHandler} serves the requested file.
 */
@Test
public void testFileServing() throws Exception {
	final Time cacheEntryDuration = Time.milliseconds(1000L);

	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(1);

	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(cacheEntryDuration, requestFileUploads, EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	assertThat(outputFile.length(), is(greaterThan(0L)));
	assertThat(FileUtils.readFileUtf8(outputFile), is(equalTo(fileContent1)));
}

Example #4

Source File: AbstractTaskManagerFileHandler.java From Flink-CEPplus with Apache License 2.0

6 votes

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<ResourceID, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(ResourceID resourceId) throws Exception {
					return loadTaskManagerFile(resourceId);
				}
		});
}

Example #5

Source File: AbstractTaskManagerFileHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}

Example #6

Source File: TaskExecutorTest.java From flink with Apache License 2.0

6 votes

@Test(timeout = 10000L)
public void testLogNotFoundHandling() throws Throwable {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		try {
			CompletableFuture<TransientBlobKey> logFuture =
				tmGateway.requestFileUpload(FileType.LOG, timeout);
			logFuture.get();
		} catch (Exception e) {
			assertThat(e.getMessage(), containsString("The file LOG does not exist on the TaskExecutor."));
		}
	}
}

Example #7

Source File: TaskExecutorTest.java From flink with Apache License 2.0

6 votes

@Test(timeout = 10000L)
public void testLogNotFoundHandling() throws Throwable {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

	try (TaskSubmissionTestEnvironment env =
		new Builder(jobId)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		try {
			CompletableFuture<TransientBlobKey> logFuture =
				tmGateway.requestFileUploadByType(FileType.LOG, timeout);
			logFuture.get();
		} catch (Exception e) {
			assertThat(e.getMessage(), containsString("The file LOG does not exist on the TaskExecutor."));
		}
	}
}

Example #8

Source File: AbstractTaskManagerFileHandler.java From flink with Apache License 2.0

6 votes

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<ResourceID, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(ResourceID resourceId) throws Exception {
					return loadTaskManagerFile(resourceId);
				}
		});
}

Example #9

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

6 votes

private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}

Example #10

Source File: AbstractTaskManagerFileHandler.java From flink with Apache License 2.0

6 votes

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws Exception {
					return loadTaskManagerFile(taskManagerIdAndFileName);
				}
		});
}

Example #11

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

6 votes

/**
 * Tests that the {@link AbstractTaskManagerFileHandler} serves the requested file.
 */
@Test
public void testFileServing() throws Exception {
	final Time cacheEntryDuration = Time.milliseconds(1000L);

	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(1);

	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(cacheEntryDuration, requestFileUploads, EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	assertThat(outputFile.length(), is(greaterThan(0L)));
	assertThat(FileUtils.readFileUtf8(outputFile), is(equalTo(fileContent1)));
}

Example #12

Source File: AbstractTaskManagerFileHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Tests that the {@link AbstractTaskManagerFileHandler} serves the requested file.
 */
@Test
public void testFileServing() throws Exception {
	final Time cacheEntryDuration = Time.milliseconds(1000L);

	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(1);

	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(cacheEntryDuration, requestFileUploads, EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	assertThat(outputFile.length(), is(greaterThan(0L)));
	assertThat(FileUtils.readFileUtf8(outputFile), is(equalTo(fileContent1)));
}

Example #13

Source File: TaskExecutor.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestFileUpload(FileType fileType, Time timeout) {
	log.debug("Request file {} upload.", fileType);

	final String filePath;

	switch (fileType) {
		case LOG:
			filePath = taskManagerConfiguration.getTaskManagerLogPath();
			break;
		case STDOUT:
			filePath = taskManagerConfiguration.getTaskManagerStdoutPath();
			break;
		default:
			filePath = null;
	}

	if (filePath != null && !filePath.isEmpty()) {
		final File file = new File(filePath);

		if (file.exists()) {
			final TransientBlobCache transientBlobService = blobCacheService.getTransientBlobService();
			final TransientBlobKey transientBlobKey;
			try (FileInputStream fileInputStream = new FileInputStream(file)) {
				transientBlobKey = transientBlobService.putTransient(fileInputStream);
			} catch (IOException e) {
				log.debug("Could not upload file {}.", fileType, e);
				return FutureUtils.completedExceptionally(new FlinkException("Could not upload file " + fileType + '.', e));
			}

			return CompletableFuture.completedFuture(transientBlobKey);
		} else {
			log.debug("The file {} does not exist on the TaskExecutor {}.", fileType, getResourceID());
			return FutureUtils.completedExceptionally(new FlinkException("The file " + fileType + " does not exist on the TaskExecutor."));
		}
	} else {
		log.debug("The file {} is unavailable on the TaskExecutor {}.", fileType, getResourceID());
		return FutureUtils.completedExceptionally(new FlinkException("The file " + fileType + " is not available on the TaskExecutor."));
	}
}

Example #14

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

private CompletableFuture<TransientBlobKey> putTransientBlobStream(InputStream inputStream, String fileTag) {
	final TransientBlobCache transientBlobService = blobCacheService.getTransientBlobService();
	final TransientBlobKey transientBlobKey;

	try {
		transientBlobKey = transientBlobService.putTransient(inputStream);
	} catch (IOException e) {
		log.debug("Could not upload file {}.", fileTag, e);
		return FutureUtils.completedExceptionally(new FlinkException("Could not upload file " + fileTag + '.', e));
	}
	return CompletableFuture.completedFuture(transientBlobKey);
}

Example #15

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

5 votes

private File runFileCachingTest(
		Time cacheEntryDuration,
		Time delayBetweenRequests) throws Exception {
	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(2);
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey2));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(
		cacheEntryDuration,
		requestFileUploads,
		EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	Thread.sleep(delayBetweenRequests.toMilliseconds());

	// the handler should not trigger the file upload again because it is still cached
	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);
	return outputFile;
}

Example #16

Source File: TestingResourceManagerGateway.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUpload(ResourceID taskManagerId, FileType fileType, Time timeout) {
	final Function<Tuple2<ResourceID, FileType>, CompletableFuture<TransientBlobKey>> function = requestTaskManagerFileUploadFunction;

	if (function != null) {
		return function.apply(Tuple2.of(taskManagerId, fileType));
	} else {
		return CompletableFuture.completedFuture(new TransientBlobKey());
	}
}

Example #17

Source File: AbstractTaskManagerFileHandler.java From flink with Apache License 2.0

5 votes

private void removeBlob(RemovalNotification<ResourceID, CompletableFuture<TransientBlobKey>> removalNotification) {
	log.debug("Remove cached file for TaskExecutor {}.", removalNotification.getKey());

	final CompletableFuture<TransientBlobKey> value = removalNotification.getValue();

	if (value != null) {
		value.thenAccept(transientBlobService::deleteFromCache);
	}
}

Example #18

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestFileUpload(FileType fileType, Time timeout) {
	log.debug("Request file {} upload.", fileType);

	final String filePath;

	switch (fileType) {
		case LOG:
			filePath = taskManagerConfiguration.getTaskManagerLogPath();
			break;
		case STDOUT:
			filePath = taskManagerConfiguration.getTaskManagerStdoutPath();
			break;
		default:
			filePath = null;
	}

	if (filePath != null && !filePath.isEmpty()) {
		final File file = new File(filePath);

		if (file.exists()) {
			final TransientBlobCache transientBlobService = blobCacheService.getTransientBlobService();
			final TransientBlobKey transientBlobKey;
			try (FileInputStream fileInputStream = new FileInputStream(file)) {
				transientBlobKey = transientBlobService.putTransient(fileInputStream);
			} catch (IOException e) {
				log.debug("Could not upload file {}.", fileType, e);
				return FutureUtils.completedExceptionally(new FlinkException("Could not upload file " + fileType + '.', e));
			}

			return CompletableFuture.completedFuture(transientBlobKey);
		} else {
			log.debug("The file {} does not exist on the TaskExecutor {}.", fileType, getResourceID());
			return FutureUtils.completedExceptionally(new FlinkException("The file " + fileType + " does not exist on the TaskExecutor."));
		}
	} else {
		log.debug("The file {} is unavailable on the TaskExecutor {}.", fileType, getResourceID());
		return FutureUtils.completedExceptionally(new FlinkException("The file " + fileType + " is not available on the TaskExecutor."));
	}
}

Example #19

Source File: TestingResourceManagerGateway.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUpload(ResourceID taskManagerId, FileType fileType, Time timeout) {
	final Function<Tuple2<ResourceID, FileType>, CompletableFuture<TransientBlobKey>> function = requestTaskManagerFileUploadFunction;

	if (function != null) {
		return function.apply(Tuple2.of(taskManagerId, fileType));
	} else {
		return CompletableFuture.completedFuture(new TransientBlobKey());
	}
}

Example #20

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestFileUploadByType(FileType fileType, Time timeout) {
	final String filePath;
	switch (fileType) {
		case LOG:
			filePath = taskManagerConfiguration.getTaskManagerLogPath();
			break;
		case STDOUT:
			filePath = taskManagerConfiguration.getTaskManagerStdoutPath();
			break;
		default:
			filePath = null;
	}
	return requestFileUploadByFilePath(filePath, fileType.toString());
}

Example #21

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestFileUploadByName(String fileName, Time timeout) {
	final String filePath;
	final String logDir = taskManagerConfiguration.getTaskManagerLogDir();
	if (StringUtils.isNullOrWhitespaceOnly(logDir) || StringUtils.isNullOrWhitespaceOnly(fileName)) {
		filePath = null;
	} else {
		filePath = new File(logDir, new File(fileName).getName()).getPath();
	}
	return requestFileUploadByFilePath(filePath, fileName);
}

Example #22

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

5 votes

@Override
protected CompletableFuture<TransientBlobKey> requestFileUpload(ResourceManagerGateway resourceManagerGateway, ResourceID taskManagerResourceId) {
	assertThat(taskManagerResourceId, is(equalTo(expectedTaskManagerId)));
	final CompletableFuture<TransientBlobKey> transientBlobKeyFuture = requestFileUploads.poll();

	if (transientBlobKeyFuture != null) {
		return transientBlobKeyFuture;
	} else {
		return FutureUtils.completedExceptionally(new FlinkException("Could not upload file."));
	}
}

Example #23

Source File: ResourceManager.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUploadByType(ResourceID taskManagerId, FileType fileType, Time timeout) {
	log.debug("Request {} file upload from TaskExecutor {}.", fileType, taskManagerId);

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(taskManagerId);

	if (taskExecutor == null) {
		log.debug("Request upload of file {} from unregistered TaskExecutor {}.", fileType, taskManagerId);
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(taskManagerId));
	} else {
		return taskExecutor.getTaskExecutorGateway().requestFileUploadByType(fileType, timeout);
	}
}

Example #24

Source File: ResourceManager.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUploadByName(ResourceID taskManagerId, String fileName, Time timeout) {
	log.debug("Request upload of file {} from TaskExecutor {}.", fileName, taskManagerId);

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(taskManagerId);

	if (taskExecutor == null) {
		log.debug("Request upload of file {} from unregistered TaskExecutor {}.", fileName, taskManagerId);
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(taskManagerId));
	} else {
		return taskExecutor.getTaskExecutorGateway().requestFileUploadByName(fileName, timeout);
	}
}

Example #25

Source File: AbstractTaskManagerFileHandler.java From flink with Apache License 2.0

5 votes

private CompletableFuture<TransientBlobKey> loadTaskManagerFile(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws RestHandlerException {
	log.debug("Load file from TaskManager {}.", taskManagerIdAndFileName.f0);

	final ResourceManagerGateway resourceManagerGateway = resourceManagerGatewayRetriever
		.getNow()
		.orElseThrow(() -> {
			log.debug("Could not connect to ResourceManager right now.");
			return new RestHandlerException(
				"Cannot connect to ResourceManager right now. Please try to refresh.",
				HttpResponseStatus.NOT_FOUND);
		});

	return requestFileUpload(resourceManagerGateway, taskManagerIdAndFileName);
}

Example #26

Source File: AbstractTaskManagerFileHandler.java From flink with Apache License 2.0

5 votes

private void removeBlob(RemovalNotification<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>> removalNotification) {
	log.debug("Remove cached file for TaskExecutor {}.", removalNotification.getKey());

	final CompletableFuture<TransientBlobKey> value = removalNotification.getValue();

	if (value != null) {
		value.thenAccept(transientBlobService::deleteFromCache);
	}
}

Example #27

Source File: TestingResourceManagerGateway.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUploadByType(ResourceID taskManagerId, FileType fileType, Time timeout) {
	final Function<Tuple2<ResourceID, FileType>, CompletableFuture<TransientBlobKey>> function = requestTaskManagerFileUploadByTypeFunction;

	if (function != null) {
		return function.apply(Tuple2.of(taskManagerId, fileType));
	} else {
		return CompletableFuture.completedFuture(new TransientBlobKey());
	}
}

Example #28

Source File: TestingResourceManagerGateway.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<TransientBlobKey> requestTaskManagerFileUploadByName(ResourceID taskManagerId, String fileName, Time timeout) {
	final Function<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>> function = requestTaskManagerFileUploadByNameFunction;

	if (function != null) {
		return function.apply(Tuple2.of(taskManagerId, fileName));
	} else {
		return CompletableFuture.completedFuture(new TransientBlobKey());
	}
}

Example #29

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

5 votes

private File runFileCachingTest(
		Time cacheEntryDuration,
		Time delayBetweenRequests) throws Exception {
	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(2);
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey2));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(
		cacheEntryDuration,
		requestFileUploads,
		EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	Thread.sleep(delayBetweenRequests.toMilliseconds());

	// the handler should not trigger the file upload again because it is still cached
	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);
	return outputFile;
}

Example #30

Source File: AbstractTaskManagerFileHandlerTest.java From flink with Apache License 2.0

5 votes

@Override
protected CompletableFuture<TransientBlobKey> requestFileUpload(ResourceManagerGateway resourceManagerGateway, Tuple2<ResourceID, String> taskManagerIdAndFileName) {
	assertThat(taskManagerIdAndFileName.f0, is(equalTo(expectedTaskManagerId)));
	final CompletableFuture<TransientBlobKey> transientBlobKeyFuture = requestFileUploads.poll();

	if (transientBlobKeyFuture != null) {
		return transientBlobKeyFuture;
	} else {
		return FutureUtils.completedExceptionally(new FlinkException("Could not upload file."));
	}
}