org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders Java Examples

The following examples show how to use org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroKryoClassloadingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"));

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 
Example #2
Source File: JobManagerRunnerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 
Example #3
Source File: AvroKryoClassloadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"));

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 
Example #4
Source File: JobManagerRunnerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLibraryCacheManagerRegistration() throws Exception {
	final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		VoidPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});
	final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager);

	try {
		jobManagerRunner.start();

		final JobID jobID = jobGraph.getJobID();
		assertThat(libraryCacheManager.hasClassLoader(jobID), is(true));

		jobManagerRunner.close();

		assertThat(libraryCacheManager.hasClassLoader(jobID), is(false));
	} finally {
		jobManagerRunner.close();
	}
}
 
Example #5
Source File: ClassLoaderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testParentFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 
Example #6
Source File: ClassLoaderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertNotEquals(clazz1, clazz2);
	assertNotEquals(clazz1, clazz3);
	assertNotEquals(clazz2, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 
Example #7
Source File: ClassLoaderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testRepeatedChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertNotEquals(clazz1, clazz2);

	assertEquals(clazz2, clazz3);
	assertEquals(clazz2, clazz4);

	childClassLoader.close();
}
 
Example #8
Source File: ClassLoaderTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testRepeatedParentFirstPatternClass() throws Exception {
	final String className = ClassLoaderTest.class.getName();
	final String parentFirstPattern = className.substring(0, className.lastIndexOf('.'));

	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern });

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);
	assertEquals(clazz1, clazz4);

	childClassLoader.close();
}
 
Example #9
Source File: ClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testRepeatedParentFirstPatternClass() throws Exception {
	final String className = ClassLoaderTest.class.getName();
	final String parentFirstPattern = className.substring(0, className.lastIndexOf('.'));

	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern });

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);
	assertEquals(clazz1, clazz4);

	childClassLoader.close();
}
 
Example #10
Source File: ClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testRepeatedChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertNotEquals(clazz1, clazz2);

	assertEquals(clazz2, clazz3);
	assertEquals(clazz2, clazz4);

	childClassLoader.close();
}
 
Example #11
Source File: ClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertNotEquals(clazz1, clazz2);
	assertNotEquals(clazz1, clazz3);
	assertNotEquals(clazz2, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 
Example #12
Source File: ClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testParentFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 
Example #13
Source File: JobManagerRunnerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testLibraryCacheManagerRegistration() throws Exception {
	final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		VoidPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});
	final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager);

	try {
		jobManagerRunner.start();

		final JobID jobID = jobGraph.getJobID();
		assertThat(libraryCacheManager.hasClassLoader(jobID), is(true));

		jobManagerRunner.close();

		assertThat(libraryCacheManager.hasClassLoader(jobID), is(false));
	} finally {
		jobManagerRunner.close();
	}
}
 
Example #14
Source File: JobManagerRunnerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 
Example #15
Source File: JobWithJars.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) {
	URL[] urls = new URL[jars.size() + classpaths.size()];
	for (int i = 0; i < jars.size(); i++) {
		urls[i] = jars.get(i);
	}
	for (int i = 0; i < classpaths.size(); i++) {
		urls[i + jars.size()] = classpaths.get(i);
	}
	return FlinkUserCodeClassLoaders.parentFirst(urls, parent);
}
 
Example #16
Source File: StatefulFunctionsJob.java    From stateful-functions with Apache License 2.0 5 votes vote down vote up
private static void setDefaultContextClassLoaderIfAbsent() {
  ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
  if (classLoader == null) {
    URLClassLoader flinkClassLoader =
        FlinkUserCodeClassLoaders.parentFirst(
            new URL[0], StatefulFunctionsJob.class.getClassLoader());
    Thread.currentThread().setContextClassLoader(flinkClassLoader);
  }
}
 
Example #17
Source File: StatefulFunctionsJob.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
private static void setDefaultContextClassLoaderIfAbsent() {
  ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
  if (classLoader == null) {
    URLClassLoader flinkClassLoader =
        FlinkUserCodeClassLoaders.parentFirst(
            new URL[0], StatefulFunctionsJob.class.getClassLoader());
    Thread.currentThread().setContextClassLoader(flinkClassLoader);
  }
}
 
Example #18
Source File: AvroKryoClassloadingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"),
		NOOP_EXCEPTION_HANDLER);

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 
Example #19
Source File: RocksDbMultiClassLoaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 
Example #20
Source File: TaskManagerConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 
Example #21
Source File: ExecutionContext.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create table sources & sinks.
	tableSources = new HashMap<>();
	tableSinks = new HashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new HashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}
 
Example #22
Source File: RocksDbMultiClassLoaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 
Example #23
Source File: RocksDbMultiClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 
Example #24
Source File: JobWithJars.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) {
	URL[] urls = new URL[jars.size() + classpaths.size()];
	for (int i = 0; i < jars.size(); i++) {
		urls[i] = jars.get(i);
	}
	for (int i = 0; i < classpaths.size(); i++) {
		urls[i + jars.size()] = classpaths.get(i);
	}
	return FlinkUserCodeClassLoaders.parentFirst(urls, parent);
}
 
Example #25
Source File: TaskManagerConfiguration.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 
Example #26
Source File: TaskTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testExecutionFailsInBlobsMissing() throws Exception {
	final PermanentBlobKey missingKey = new PermanentBlobKey();

	final Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		TEMPORARY_FOLDER.newFolder().getAbsolutePath());
	config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	final BlobServer blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
	InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
	final PermanentBlobCache permanentBlobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress);

	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			permanentBlobCache,
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]);

	final Task task = new TaskBuilder()
		.setRequiredJarFileBlobKeys(Collections.singletonList(missingKey))
		.setLibraryCacheManager(libraryCacheManager)
		.build();

	// task should be new and perfect
	assertEquals(ExecutionState.CREATED, task.getExecutionState());
	assertFalse(task.isCanceledOrFailed());
	assertNull(task.getFailureCause());

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertNotNull(task.getFailureCause());
	assertNotNull(task.getFailureCause().getMessage());
	assertTrue(task.getFailureCause().getMessage().contains("Failed to fetch BLOB"));

	assertNull(task.getInvokable());
}
 
Example #27
Source File: TaskManagerServicesConfiguration.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}
 
Example #28
Source File: TaskManagerServicesConfiguration.java    From flink with Apache License 2.0 4 votes vote down vote up
public FlinkUserCodeClassLoaders.ResolveOrder getClassLoaderResolveOrder() {
	return classLoaderResolveOrder;
}
 
Example #29
Source File: TaskManagerServicesConfiguration.java    From flink with Apache License 2.0 4 votes vote down vote up
private TaskManagerServicesConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		InetAddress bindAddress,
		int externalDataPort,
		boolean localCommunicationOnly,
		String[] tmpDirPaths,
		String[] localRecoveryStateRootDirectories,
		boolean localRecoveryEnabled,
		@Nullable QueryableStateConfiguration queryableStateConfig,
		int numberOfSlots,
		int pageSize,
		TaskExecutorResourceSpec taskExecutorResourceSpec,
		long timerServiceShutdownTimeout,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Optional<Time> systemResourceMetricsProbingInterval,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		int numIoThreads) {
	this.configuration = checkNotNull(configuration);
	this.resourceID = checkNotNull(resourceID);

	this.externalAddress = checkNotNull(externalAddress);
	this.bindAddress = checkNotNull(bindAddress);
	this.externalDataPort = externalDataPort;
	this.localCommunicationOnly = localCommunicationOnly;
	this.tmpDirPaths = checkNotNull(tmpDirPaths);
	this.localRecoveryStateRootDirectories = checkNotNull(localRecoveryStateRootDirectories);
	this.localRecoveryEnabled = checkNotNull(localRecoveryEnabled);
	this.queryableStateConfig = queryableStateConfig;
	this.numberOfSlots = checkNotNull(numberOfSlots);

	this.pageSize = pageSize;

	this.taskExecutorResourceSpec = taskExecutorResourceSpec;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.numIoThreads = numIoThreads;

	checkArgument(timerServiceShutdownTimeout >= 0L, "The timer " +
		"service shutdown timeout must be greater or equal to 0.");
	this.timerServiceShutdownTimeout = timerServiceShutdownTimeout;
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);

	this.systemResourceMetricsProbingInterval = checkNotNull(systemResourceMetricsProbingInterval);
}
 
Example #30
Source File: JobManagerSharedServices.java    From flink with Apache License 2.0 4 votes vote down vote up
public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer,
		FatalErrorHandler fatalErrorHandler) {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			BlobLibraryCacheManager.defaultClassLoaderFactory(
				FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
				alwaysParentFirstLoaderPatterns,
				failOnJvmMetaspaceOomError ? fatalErrorHandler : null));

	final Duration akkaTimeout;
	try {
		akkaTimeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final int numSamples = config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES);
	final long delayBetweenSamples = config.getInteger(WebOptions.BACKPRESSURE_DELAY);
	final BackPressureRequestCoordinator coordinator = new BackPressureRequestCoordinator(
		futureExecutor,
		akkaTimeout.toMillis() + numSamples * delayBetweenSamples);

	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		coordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		coordinator,
		backPressureStatsTracker,
		blobServer);
}