org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders Java Examples
The following examples show how to use
org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroKryoClassloadingTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testKryoInChildClasspath() throws Exception { final Class<?> avroClass = AvroKryoSerializerUtils.class; final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation(); final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parentClassLoader = new FilteredClassLoader( avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName()); final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { avroLocation, kryoLocation }, parentClassLoader, CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";")); final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader); assertNotEquals(avroClass, userLoadedAvroClass); // call the 'addAvroGenericDataArrayRegistration(...)' method final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class); final LinkedHashMap<String, ?> map = new LinkedHashMap<>(); m.invoke(userLoadedAvroClass.newInstance(), map); assertEquals(1, map.size()); }
Example #2
Source File: JobManagerRunnerTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void setupClass() { libraryCacheManager = new BlobLibraryCacheManager( FailingPermanentBlobService.INSTANCE, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[]{}); defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory(); final JobVertex jobVertex = new JobVertex("Test vertex"); jobVertex.setInvokableClass(NoOpInvokable.class); jobGraph = new JobGraph(jobVertex); archivedExecutionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobGraph.getJobID()) .setState(JobStatus.FINISHED) .build(); }
Example #3
Source File: AvroKryoClassloadingTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testKryoInChildClasspath() throws Exception { final Class<?> avroClass = AvroKryoSerializerUtils.class; final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation(); final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parentClassLoader = new FilteredClassLoader( avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName()); final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { avroLocation, kryoLocation }, parentClassLoader, CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";")); final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader); assertNotEquals(avroClass, userLoadedAvroClass); // call the 'addAvroGenericDataArrayRegistration(...)' method final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class); final LinkedHashMap<String, ?> map = new LinkedHashMap<>(); m.invoke(userLoadedAvroClass.newInstance(), map); assertEquals(1, map.size()); }
Example #4
Source File: JobManagerRunnerTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testLibraryCacheManagerRegistration() throws Exception { final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager( VoidPermanentBlobService.INSTANCE, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[]{}); final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager); try { jobManagerRunner.start(); final JobID jobID = jobGraph.getJobID(); assertThat(libraryCacheManager.hasClassLoader(jobID), is(true)); jobManagerRunner.close(); assertThat(libraryCacheManager.hasClassLoader(jobID), is(false)); } finally { jobManagerRunner.close(); } }
Example #5
Source File: ClassLoaderTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testParentFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst( new URL[] { childCodePath }, parentClassLoader); final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst( new URL[] { childCodePath }, parentClassLoader); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader1); final Class<?> clazz3 = Class.forName(className, false, childClassLoader2); assertEquals(clazz1, clazz2); assertEquals(clazz1, clazz3); childClassLoader1.close(); childClassLoader2.close(); }
Example #6
Source File: ClassLoaderTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testChildFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader1); final Class<?> clazz3 = Class.forName(className, false, childClassLoader2); assertNotEquals(clazz1, clazz2); assertNotEquals(clazz1, clazz3); assertNotEquals(clazz2, clazz3); childClassLoader1.close(); childClassLoader2.close(); }
Example #7
Source File: ClassLoaderTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testRepeatedChildFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader); final Class<?> clazz3 = Class.forName(className, false, childClassLoader); final Class<?> clazz4 = Class.forName(className, false, childClassLoader); assertNotEquals(clazz1, clazz2); assertEquals(clazz2, clazz3); assertEquals(clazz2, clazz4); childClassLoader.close(); }
Example #8
Source File: ClassLoaderTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testRepeatedParentFirstPatternClass() throws Exception { final String className = ClassLoaderTest.class.getName(); final String parentFirstPattern = className.substring(0, className.lastIndexOf('.')); final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern }); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader); final Class<?> clazz3 = Class.forName(className, false, childClassLoader); final Class<?> clazz4 = Class.forName(className, false, childClassLoader); assertEquals(clazz1, clazz2); assertEquals(clazz1, clazz3); assertEquals(clazz1, clazz4); childClassLoader.close(); }
Example #9
Source File: ClassLoaderTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testRepeatedParentFirstPatternClass() throws Exception { final String className = ClassLoaderTest.class.getName(); final String parentFirstPattern = className.substring(0, className.lastIndexOf('.')); final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern }); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader); final Class<?> clazz3 = Class.forName(className, false, childClassLoader); final Class<?> clazz4 = Class.forName(className, false, childClassLoader); assertEquals(clazz1, clazz2); assertEquals(clazz1, clazz3); assertEquals(clazz1, clazz4); childClassLoader.close(); }
Example #10
Source File: ClassLoaderTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testRepeatedChildFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader); final Class<?> clazz3 = Class.forName(className, false, childClassLoader); final Class<?> clazz4 = Class.forName(className, false, childClassLoader); assertNotEquals(clazz1, clazz2); assertEquals(clazz2, clazz3); assertEquals(clazz2, clazz4); childClassLoader.close(); }
Example #11
Source File: ClassLoaderTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testChildFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst( new URL[] { childCodePath }, parentClassLoader, new String[0]); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader1); final Class<?> clazz3 = Class.forName(className, false, childClassLoader2); assertNotEquals(clazz1, clazz2); assertNotEquals(clazz1, clazz3); assertNotEquals(clazz2, clazz3); childClassLoader1.close(); childClassLoader2.close(); }
Example #12
Source File: ClassLoaderTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testParentFirstClassLoading() throws Exception { final ClassLoader parentClassLoader = getClass().getClassLoader(); // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation(); final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst( new URL[] { childCodePath }, parentClassLoader); final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst( new URL[] { childCodePath }, parentClassLoader); final String className = ClassLoaderTest.class.getName(); final Class<?> clazz1 = Class.forName(className, false, parentClassLoader); final Class<?> clazz2 = Class.forName(className, false, childClassLoader1); final Class<?> clazz3 = Class.forName(className, false, childClassLoader2); assertEquals(clazz1, clazz2); assertEquals(clazz1, clazz3); childClassLoader1.close(); childClassLoader2.close(); }
Example #13
Source File: JobManagerRunnerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testLibraryCacheManagerRegistration() throws Exception { final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager( VoidPermanentBlobService.INSTANCE, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[]{}); final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager); try { jobManagerRunner.start(); final JobID jobID = jobGraph.getJobID(); assertThat(libraryCacheManager.hasClassLoader(jobID), is(true)); jobManagerRunner.close(); assertThat(libraryCacheManager.hasClassLoader(jobID), is(false)); } finally { jobManagerRunner.close(); } }
Example #14
Source File: JobManagerRunnerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@BeforeClass public static void setupClass() { libraryCacheManager = new BlobLibraryCacheManager( FailingPermanentBlobService.INSTANCE, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[]{}); defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory(); final JobVertex jobVertex = new JobVertex("Test vertex"); jobVertex.setInvokableClass(NoOpInvokable.class); jobGraph = new JobGraph(jobVertex); archivedExecutionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobGraph.getJobID()) .setState(JobStatus.FINISHED) .build(); }
Example #15
Source File: JobWithJars.java From flink with Apache License 2.0 | 5 votes |
public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) { URL[] urls = new URL[jars.size() + classpaths.size()]; for (int i = 0; i < jars.size(); i++) { urls[i] = jars.get(i); } for (int i = 0; i < classpaths.size(); i++) { urls[i + jars.size()] = classpaths.get(i); } return FlinkUserCodeClassLoaders.parentFirst(urls, parent); }
Example #16
Source File: StatefulFunctionsJob.java From stateful-functions with Apache License 2.0 | 5 votes |
private static void setDefaultContextClassLoaderIfAbsent() { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { URLClassLoader flinkClassLoader = FlinkUserCodeClassLoaders.parentFirst( new URL[0], StatefulFunctionsJob.class.getClassLoader()); Thread.currentThread().setContextClassLoader(flinkClassLoader); } }
Example #17
Source File: StatefulFunctionsJob.java From flink-statefun with Apache License 2.0 | 5 votes |
private static void setDefaultContextClassLoaderIfAbsent() { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { URLClassLoader flinkClassLoader = FlinkUserCodeClassLoaders.parentFirst( new URL[0], StatefulFunctionsJob.class.getClassLoader()); Thread.currentThread().setContextClassLoader(flinkClassLoader); } }
Example #18
Source File: AvroKryoClassloadingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testKryoInChildClasspath() throws Exception { final Class<?> avroClass = AvroKryoSerializerUtils.class; final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation(); final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parentClassLoader = new FilteredClassLoader( avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName()); final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { avroLocation, kryoLocation }, parentClassLoader, CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"), NOOP_EXCEPTION_HANDLER); final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader); assertNotEquals(avroClass, userLoadedAvroClass); // call the 'addAvroGenericDataArrayRegistration(...)' method final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class); final LinkedHashMap<String, ?> map = new LinkedHashMap<>(); m.invoke(userLoadedAvroClass.newInstance(), map); assertEquals(1, map.size()); }
Example #19
Source File: RocksDbMultiClassLoaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTwoSeparateClassLoaders() throws Exception { // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation(); final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parent = getClass().getClassLoader(); final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER); final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER); final String className = RocksDBStateBackend.class.getName(); final Class<?> clazz1 = Class.forName(className, false, loader1); final Class<?> clazz2 = Class.forName(className, false, loader2); assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2); final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final String tempDir = tmp.newFolder().getAbsolutePath(); final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); meth1.setAccessible(true); meth2.setAccessible(true); // if all is well, these methods can both complete successfully meth1.invoke(instance1, tempDir); meth2.invoke(instance2, tempDir); }
Example #20
Source File: TaskManagerConfiguration.java From flink with Apache License 2.0 | 5 votes |
public TaskManagerConfiguration( int numberSlots, String[] tmpDirectories, Time timeout, @Nullable Time maxRegistrationDuration, Time initialRegistrationPause, Time maxRegistrationPause, Time refusedRegistrationPause, Configuration configuration, boolean exitJvmOnOutOfMemory, FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder, String[] alwaysParentFirstLoaderPatterns, @Nullable String taskManagerLogPath, @Nullable String taskManagerStdoutPath, RetryingRegistrationConfiguration retryingRegistrationConfiguration) { this.numberSlots = numberSlots; this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories); this.timeout = Preconditions.checkNotNull(timeout); this.maxRegistrationDuration = maxRegistrationDuration; this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause); this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause); this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause); this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration)); this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory; this.classLoaderResolveOrder = classLoaderResolveOrder; this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns; this.taskManagerLogPath = taskManagerLogPath; this.taskManagerStdoutPath = taskManagerStdoutPath; this.retryingRegistrationConfiguration = retryingRegistrationConfiguration; }
Example #21
Source File: ExecutionContext.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies, Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) { this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment()); this.dependencies = dependencies; this.flinkConfig = flinkConfig; // create class loader classLoader = FlinkUserCodeClassLoaders.parentFirst( dependencies.toArray(new URL[dependencies.size()]), this.getClass().getClassLoader()); // create table sources & sinks. tableSources = new HashMap<>(); tableSinks = new HashMap<>(); mergedEnv.getTables().forEach((name, entry) -> { if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) { tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader)); } if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) { tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader)); } }); // create user-defined functions functions = new HashMap<>(); mergedEnv.getFunctions().forEach((name, entry) -> { final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false); functions.put(name, function); }); // convert deployment options into command line options that describe a cluster commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions); activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine); runOptions = createRunOptions(commandLine); clusterId = activeCommandLine.getClusterId(commandLine); clusterSpec = createClusterSpecification(activeCommandLine, commandLine); }
Example #22
Source File: RocksDbMultiClassLoaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTwoSeparateClassLoaders() throws Exception { // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation(); final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parent = getClass().getClassLoader(); final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]); final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]); final String className = RocksDBStateBackend.class.getName(); final Class<?> clazz1 = Class.forName(className, false, loader1); final Class<?> clazz2 = Class.forName(className, false, loader2); assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2); final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final String tempDir = tmp.newFolder().getAbsolutePath(); final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); meth1.setAccessible(true); meth2.setAccessible(true); // if all is well, these methods can both complete successfully meth1.invoke(instance1, tempDir); meth2.invoke(instance2, tempDir); }
Example #23
Source File: RocksDbMultiClassLoaderTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testTwoSeparateClassLoaders() throws Exception { // collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation(); final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation(); final ClassLoader parent = getClass().getClassLoader(); final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]); final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]); final String className = RocksDBStateBackend.class.getName(); final Class<?> clazz1 = Class.forName(className, false, loader1); final Class<?> clazz2 = Class.forName(className, false, loader2); assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2); final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString()); final String tempDir = tmp.newFolder().getAbsolutePath(); final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); meth1.setAccessible(true); meth2.setAccessible(true); // if all is well, these methods can both complete successfully meth1.invoke(instance1, tempDir); meth2.invoke(instance2, tempDir); }
Example #24
Source File: JobWithJars.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) { URL[] urls = new URL[jars.size() + classpaths.size()]; for (int i = 0; i < jars.size(); i++) { urls[i] = jars.get(i); } for (int i = 0; i < classpaths.size(); i++) { urls[i + jars.size()] = classpaths.get(i); } return FlinkUserCodeClassLoaders.parentFirst(urls, parent); }
Example #25
Source File: TaskManagerConfiguration.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public TaskManagerConfiguration( int numberSlots, String[] tmpDirectories, Time timeout, @Nullable Time maxRegistrationDuration, Time initialRegistrationPause, Time maxRegistrationPause, Time refusedRegistrationPause, Configuration configuration, boolean exitJvmOnOutOfMemory, FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder, String[] alwaysParentFirstLoaderPatterns, @Nullable String taskManagerLogPath, @Nullable String taskManagerStdoutPath, RetryingRegistrationConfiguration retryingRegistrationConfiguration) { this.numberSlots = numberSlots; this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories); this.timeout = Preconditions.checkNotNull(timeout); this.maxRegistrationDuration = maxRegistrationDuration; this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause); this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause); this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause); this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration)); this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory; this.classLoaderResolveOrder = classLoaderResolveOrder; this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns; this.taskManagerLogPath = taskManagerLogPath; this.taskManagerStdoutPath = taskManagerStdoutPath; this.retryingRegistrationConfiguration = retryingRegistrationConfiguration; }
Example #26
Source File: TaskTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testExecutionFailsInBlobsMissing() throws Exception { final PermanentBlobKey missingKey = new PermanentBlobKey(); final Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, TEMPORARY_FOLDER.newFolder().getAbsolutePath()); config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L); final BlobServer blobServer = new BlobServer(config, new VoidBlobStore()); blobServer.start(); InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort()); final PermanentBlobCache permanentBlobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress); final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager( permanentBlobCache, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]); final Task task = new TaskBuilder() .setRequiredJarFileBlobKeys(Collections.singletonList(missingKey)) .setLibraryCacheManager(libraryCacheManager) .build(); // task should be new and perfect assertEquals(ExecutionState.CREATED, task.getExecutionState()); assertFalse(task.isCanceledOrFailed()); assertNull(task.getFailureCause()); // should fail task.run(); // verify final state assertEquals(ExecutionState.FAILED, task.getExecutionState()); assertTrue(task.isCanceledOrFailed()); assertNotNull(task.getFailureCause()); assertNotNull(task.getFailureCause().getMessage()); assertTrue(task.getFailureCause().getMessage().contains("Failed to fetch BLOB")); assertNull(task.getInvokable()); }
Example #27
Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0 | 4 votes |
/** * Utility method to extract TaskManager config parameters from the configuration and to * sanity check them. * * @param configuration The configuration. * @param resourceID resource ID of the task manager * @param externalAddress identifying the IP address under which the TaskManager will be accessible * @param localCommunicationOnly True if only local communication is possible. * Use only in cases where only one task manager runs. * * @return configuration of task manager services used to create them */ public static TaskManagerServicesConfiguration fromConfiguration( Configuration configuration, ResourceID resourceID, String externalAddress, boolean localCommunicationOnly, TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception { final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration); String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration); if (localStateRootDir.length == 0) { // default to temp dirs. localStateRootDir = tmpDirs; } boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY); final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration); long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis(); final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration); final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT); String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress()); InetAddress bindAddress = InetAddress.getByName(bindAddr); final String classLoaderResolveOrder = configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER); final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration); final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration); return new TaskManagerServicesConfiguration( configuration, resourceID, externalAddress, bindAddress, externalDataPort, localCommunicationOnly, tmpDirs, localStateRootDir, localRecoveryMode, queryableStateConfig, ConfigurationParserUtils.getSlot(configuration), ConfigurationParserUtils.getPageSize(configuration), taskExecutorResourceSpec, timerServiceShutdownTimeout, retryingRegistrationConfiguration, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration), FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns, numIoThreads); }
Example #28
Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0 | 4 votes |
public FlinkUserCodeClassLoaders.ResolveOrder getClassLoaderResolveOrder() { return classLoaderResolveOrder; }
Example #29
Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0 | 4 votes |
private TaskManagerServicesConfiguration( Configuration configuration, ResourceID resourceID, String externalAddress, InetAddress bindAddress, int externalDataPort, boolean localCommunicationOnly, String[] tmpDirPaths, String[] localRecoveryStateRootDirectories, boolean localRecoveryEnabled, @Nullable QueryableStateConfiguration queryableStateConfig, int numberOfSlots, int pageSize, TaskExecutorResourceSpec taskExecutorResourceSpec, long timerServiceShutdownTimeout, RetryingRegistrationConfiguration retryingRegistrationConfiguration, Optional<Time> systemResourceMetricsProbingInterval, FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder, String[] alwaysParentFirstLoaderPatterns, int numIoThreads) { this.configuration = checkNotNull(configuration); this.resourceID = checkNotNull(resourceID); this.externalAddress = checkNotNull(externalAddress); this.bindAddress = checkNotNull(bindAddress); this.externalDataPort = externalDataPort; this.localCommunicationOnly = localCommunicationOnly; this.tmpDirPaths = checkNotNull(tmpDirPaths); this.localRecoveryStateRootDirectories = checkNotNull(localRecoveryStateRootDirectories); this.localRecoveryEnabled = checkNotNull(localRecoveryEnabled); this.queryableStateConfig = queryableStateConfig; this.numberOfSlots = checkNotNull(numberOfSlots); this.pageSize = pageSize; this.taskExecutorResourceSpec = taskExecutorResourceSpec; this.classLoaderResolveOrder = classLoaderResolveOrder; this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns; this.numIoThreads = numIoThreads; checkArgument(timerServiceShutdownTimeout >= 0L, "The timer " + "service shutdown timeout must be greater or equal to 0."); this.timerServiceShutdownTimeout = timerServiceShutdownTimeout; this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration); this.systemResourceMetricsProbingInterval = checkNotNull(systemResourceMetricsProbingInterval); }
Example #30
Source File: JobManagerSharedServices.java From flink with Apache License 2.0 | 4 votes |
public static JobManagerSharedServices fromConfiguration( Configuration config, BlobServer blobServer, FatalErrorHandler fatalErrorHandler) { checkNotNull(config); checkNotNull(blobServer); final String classLoaderResolveOrder = config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER); final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config); final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM); final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager( blobServer, BlobLibraryCacheManager.defaultClassLoaderFactory( FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns, failOnJvmMetaspaceOomError ? fatalErrorHandler : null)); final Duration akkaTimeout; try { akkaTimeout = AkkaUtils.getTimeout(config); } catch (NumberFormatException e) { throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage()); } final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool( Hardware.getNumberCPUCores(), new ExecutorThreadFactory("jobmanager-future")); final int numSamples = config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES); final long delayBetweenSamples = config.getInteger(WebOptions.BACKPRESSURE_DELAY); final BackPressureRequestCoordinator coordinator = new BackPressureRequestCoordinator( futureExecutor, akkaTimeout.toMillis() + numSamples * delayBetweenSamples); final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL); final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl( coordinator, cleanUpInterval, config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL)); futureExecutor.scheduleWithFixedDelay( backPressureStatsTracker::cleanUpOperatorStatsCache, cleanUpInterval, cleanUpInterval, TimeUnit.MILLISECONDS); return new JobManagerSharedServices( futureExecutor, libraryCacheManager, coordinator, backPressureStatsTracker, blobServer); }