org.apache.flink.util.OperatingSystem Java Examples
The following examples show how to use
org.apache.flink.util.OperatingSystem.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BucketingSinkTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void createHDFS() throws IOException { Assume.assumeTrue("HDFS cluster cannot be started on Windows without extensions.", !OperatingSystem.isWindows()); Configuration conf = new Configuration(); File dataDir = tempFolder.newFolder(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); hdfsCluster = builder.build(); dfs = hdfsCluster.getFileSystem(); hdfsURI = "hdfs://" + NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort()) + "/"; }
Example #2
Source File: EnvironmentInformation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tries to retrieve the maximum number of open file handles. This method will only work on * UNIX-based operating systems with Sun/Oracle Java versions. * * <p>If the number of max open file handles cannot be determined, this method returns {@code -1}.</p> * * @return The limit of open file handles, or {@code -1}, if the limit could not be determined. */ public static long getOpenFileHandlesLimit() { if (OperatingSystem.isWindows()) { // getMaxFileDescriptorCount method is not available on Windows return -1L; } Class<?> sunBeanClass; try { sunBeanClass = Class.forName("com.sun.management.UnixOperatingSystemMXBean"); } catch (ClassNotFoundException e) { return -1L; } try { Method fhLimitMethod = sunBeanClass.getMethod("getMaxFileDescriptorCount"); Object result = fhLimitMethod.invoke(ManagementFactory.getOperatingSystemMXBean()); return (Long) result; } catch (Throwable t) { LOG.warn("Unexpected error when accessing file handle limit", t); return -1L; } }
Example #3
Source File: PythonEnvironmentManagerUtils.java From flink with Apache License 2.0 | 6 votes |
public static String getPythonUdfRunnerScript( String pythonExecutable, Map<String, String> environmentVariables) throws IOException { String runnerDir; if (environmentVariables.containsKey(PYFLINK_UDF_RUNNER_DIR)) { runnerDir = environmentVariables.get(PYFLINK_UDF_RUNNER_DIR); } else { String[] commands = new String[] { pythonExecutable, "-c", GET_RUNNER_DIR_SCRIPT}; String out = execute(commands, environmentVariables, false); runnerDir = out.trim(); } String runnerScriptPath; if (OperatingSystem.isWindows()) { runnerScriptPath = String.join(File.separator, runnerDir, PYFLINK_UDF_RUNNER_BAT); } else { runnerScriptPath = String.join(File.separator, runnerDir, PYFLINK_UDF_RUNNER_SH); } return runnerScriptPath; }
Example #4
Source File: PythonEnvUtilsTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSetPythonExecutable() throws IOException { Configuration config = new Configuration(); PythonEnvUtils.PythonEnvironment env = preparePythonEnvironment(config, null, tmpDirPath); if (OperatingSystem.isWindows()) { Assert.assertEquals("python.exe", env.pythonExec); } else { Assert.assertEquals("python", env.pythonExec); } Map<String, String> systemEnv = new HashMap<>(System.getenv()); systemEnv.put(PYFLINK_CLIENT_EXECUTABLE, "python3"); CommonTestUtils.setEnv(systemEnv); try { env = preparePythonEnvironment(config, null, tmpDirPath); Assert.assertEquals("python3", env.pythonExec); } finally { systemEnv.remove(PYFLINK_CLIENT_EXECUTABLE); CommonTestUtils.setEnv(systemEnv); } config.set(PYTHON_CLIENT_EXECUTABLE, "/usr/bin/python"); env = preparePythonEnvironment(config, null, tmpDirPath); Assert.assertEquals("/usr/bin/python", env.pythonExec); }
Example #5
Source File: JvmExitOnFatalErrorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testExitJvmOnOutOfMemory() throws Exception { // this test works only on linux assumeTrue(OperatingSystem.isLinux()); // to check what went wrong (when the test hangs) uncomment this line // ProcessEntryPoint.main(new String[0]); final KillOnFatalErrorProcess testProcess = new KillOnFatalErrorProcess(); try { testProcess.startProcess(); testProcess.waitFor(); } finally { testProcess.destroy(); } }
Example #6
Source File: BucketingSinkTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void createHDFS() throws IOException { Assume.assumeTrue("HDFS cluster cannot be started on Windows without extensions.", !OperatingSystem.isWindows()); Configuration conf = new Configuration(); File dataDir = tempFolder.newFolder(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); hdfsCluster = builder.build(); dfs = hdfsCluster.getFileSystem(); hdfsURI = "hdfs://" + NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort()) + "/"; }
Example #7
Source File: PythonDependencyInfoTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testParsePythonRequirements() throws IOException { // Skip this test on Windows as we can not control the Window Driver letters. Assume.assumeFalse(OperatingSystem.isWindows()); Configuration config = new Configuration(); config.set(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE, new HashMap<>()); config.get(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE) .put(PythonDependencyUtils.FILE, "python_requirements_file_{SHA256}"); PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache); assertEquals("/distributed_cache/file2", dependencyInfo.getRequirementsFilePath().get()); assertFalse(dependencyInfo.getRequirementsCacheDir().isPresent()); config.get(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE) .put(PythonDependencyUtils.CACHE, "python_requirements_cache_{SHA256}"); dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache); assertEquals("/distributed_cache/file2", dependencyInfo.getRequirementsFilePath().get()); assertEquals("/distributed_cache/file3", dependencyInfo.getRequirementsCacheDir().get()); }
Example #8
Source File: EnvironmentInformation.java From flink with Apache License 2.0 | 6 votes |
/** * Tries to retrieve the maximum number of open file handles. This method will only work on * UNIX-based operating systems with Sun/Oracle Java versions. * * <p>If the number of max open file handles cannot be determined, this method returns {@code -1}.</p> * * @return The limit of open file handles, or {@code -1}, if the limit could not be determined. */ public static long getOpenFileHandlesLimit() { if (OperatingSystem.isWindows()) { // getMaxFileDescriptorCount method is not available on Windows return -1L; } Class<?> sunBeanClass; try { sunBeanClass = Class.forName("com.sun.management.UnixOperatingSystemMXBean"); } catch (ClassNotFoundException e) { return -1L; } try { Method fhLimitMethod = sunBeanClass.getMethod("getMaxFileDescriptorCount"); Object result = fhLimitMethod.invoke(ManagementFactory.getOperatingSystemMXBean()); return (Long) result; } catch (Throwable t) { LOG.warn("Unexpected error when accessing file handle limit", t); return -1L; } }
Example #9
Source File: JvmExitOnFatalErrorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testExitJvmOnOutOfMemory() throws Exception { // this test works only on linux assumeTrue(OperatingSystem.isLinux()); // to check what went wrong (when the test hangs) uncomment this line // ProcessEntryPoint.main(new String[0]); final KillOnFatalErrorProcess testProcess = new KillOnFatalErrorProcess(); try { testProcess.startProcess(); testProcess.waitFor(); } finally { testProcess.destroy(); } }
Example #10
Source File: PythonDependencyInfoTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testParsePythonArchives() { // Skip this test on Windows as we can not control the Window Driver letters. Assume.assumeFalse(OperatingSystem.isWindows()); Configuration config = new Configuration(); Map<String, String> pythonArchives = new HashMap<>(); pythonArchives.put("python_archive_{SHA256_0}", "py27.zip"); pythonArchives.put("python_archive_{SHA256_1}", "py37"); config.set(PythonDependencyUtils.PYTHON_ARCHIVES, pythonArchives); PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache); Map<String, String> expected = new HashMap<>(); expected.put("/distributed_cache/file4", "py27.zip"); expected.put("/distributed_cache/file5", "py37"); assertEquals(expected, dependencyInfo.getArchives()); }
Example #11
Source File: ContinuousFileProcessingTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void createHDFS() { Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows()); try { File hdfsDir = tempFolder.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath()); hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set. MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/"; hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf); } catch (Throwable e) { e.printStackTrace(); Assert.fail("Test failed " + e.getMessage()); } }
Example #12
Source File: ContinuousFileProcessingTest.java From flink with Apache License 2.0 | 6 votes |
@BeforeClass public static void createHDFS() { Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows()); try { File hdfsDir = tempFolder.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath()); hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set. MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/"; hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf); } catch (Throwable e) { e.printStackTrace(); Assert.fail("Test failed " + e.getMessage()); } }
Example #13
Source File: BlobUtilsNonWritableTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Before public void before() throws IOException { assumeTrue(!OperatingSystem.isWindows()); //setWritable doesn't work on Windows. // Prepare test directory blobUtilsTestDirectory = temporaryFolder.newFolder(); assertTrue(blobUtilsTestDirectory.setExecutable(true, false)); assertTrue(blobUtilsTestDirectory.setReadable(true, false)); assertTrue(blobUtilsTestDirectory.setWritable(false, false)); }
Example #14
Source File: GlobFilePathFilterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testExcludeFilenameWithStart() { Assume.assumeTrue("Windows does not allow asterisks in file names.", !OperatingSystem.isWindows()); GlobFilePathFilter matcher = new GlobFilePathFilter( Collections.singletonList("**"), Collections.singletonList("\\*")); assertTrue(matcher.filterPath(new Path("*"))); assertFalse(matcher.filterPath(new Path("**"))); assertFalse(matcher.filterPath(new Path("other.txt"))); }
Example #15
Source File: YarnIntraNonHaMasterServicesTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue(!OperatingSystem.isWindows()); final File tempDir = TEMP_DIR.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path(hdfsCluster.getURI()); }
Example #16
Source File: YarnPreConfiguredMasterHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue(!OperatingSystem.isWindows()); final File tempDir = TEMP_DIR.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path(hdfsCluster.getURI()); }
Example #17
Source File: SignalHandler.java From flink with Apache License 2.0 | 5 votes |
/** * Register some signal handlers. * * @param LOG The slf4j logger */ public static void register(final Logger LOG) { synchronized (SignalHandler.class) { if (registered) { return; } registered = true; final String[] SIGNALS = OperatingSystem.isWindows() ? new String[]{ "TERM", "INT"} : new String[]{ "TERM", "HUP", "INT" }; StringBuilder bld = new StringBuilder(); bld.append("Registered UNIX signal handlers for ["); String separator = ""; for (String signalName : SIGNALS) { try { new Handler(signalName, LOG); bld.append(separator); bld.append(signalName); separator = ", "; } catch (Exception e) { LOG.info("Error while registering signal handler", e); } } bld.append("]"); LOG.info(bld.toString()); } }
Example #18
Source File: YarnFileStageTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue(!OperatingSystem.isWindows()); final File tempDir = CLASS_TEMP_DIR.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path(hdfsCluster.getURI()); }
Example #19
Source File: BlobServerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Nonnull private File createNonWritableDirectory() throws IOException { assumeFalse(OperatingSystem.isWindows()); //setWritable doesn't work on Windows. final File blobStorageDirectory = temporaryFolder.newFolder(); assertTrue(blobStorageDirectory.setExecutable(true, false)); assertTrue(blobStorageDirectory.setReadable(true, false)); assertTrue(blobStorageDirectory.setWritable(false, false)); return blobStorageDirectory; }
Example #20
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that Yarn will restart a killed {@link YarnSessionClusterEntrypoint} which will then resume * a persisted {@link JobGraph}. */ @Test public void testKillYarnSessionClusterEntrypoint() throws Exception { runTest(() -> { assumeTrue( "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.", OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris()); final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor(); final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor); try { final JobID jobId = submitJob(restClusterClient); final ApplicationId id = restClusterClient.getClusterId(); waitUntilJobIsRunning(restClusterClient, jobId); killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint()); waitForApplicationAttempt(id, 2); waitForJobTermination(restClusterClient, jobId); killApplicationAndWait(id); } finally { restClusterClient.close(); } }); }
Example #21
Source File: BlobServerPutTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Uploads a byte array to a server which cannot create any files via the {@link BlobServer}. * File transfers should fail. * * @param jobId * job id * @param blobType * whether the BLOB should become permanent or transient */ private void testPutBufferFails(@Nullable final JobID jobId, BlobKey.BlobType blobType) throws IOException { assumeTrue(!OperatingSystem.isWindows()); //setWritable doesn't work on Windows. final Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); File tempFileDir = null; try (BlobServer server = new BlobServer(config, new VoidBlobStore())) { server.start(); // make sure the blob server cannot create any files in its storage dir tempFileDir = server.createTemporaryFilename().getParentFile().getParentFile(); assertTrue(tempFileDir.setExecutable(true, false)); assertTrue(tempFileDir.setReadable(true, false)); assertTrue(tempFileDir.setWritable(false, false)); byte[] data = new byte[2000000]; rnd.nextBytes(data); // upload the file to the server directly exception.expect(AccessDeniedException.class); put(server, jobId, data, blobType); } finally { // set writable again to make sure we can remove the directory if (tempFileDir != null) { //noinspection ResultOfMethodCallIgnored tempFileDir.setWritable(true, false); } } }
Example #22
Source File: BlobCachePutTest.java From flink with Apache License 2.0 | 5 votes |
/** * Uploads a byte array to a server which cannot create any files via the {@link * BlobCacheService}. File transfers should fail. * * @param jobId * job id * @param blobType * whether the BLOB should become permanent or transient */ private void testPutBufferFails(@Nullable final JobID jobId, BlobKey.BlobType blobType) throws IOException { assumeTrue(!OperatingSystem.isWindows()); //setWritable doesn't work on Windows. final Configuration config = new Configuration(); config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); File tempFileDir = null; try ( BlobServer server = new BlobServer(config, new VoidBlobStore()); BlobCacheService cache = new BlobCacheService(config, new VoidBlobStore(), new InetSocketAddress("localhost", server.getPort()) )) { server.start(); // make sure the blob server cannot create any files in its storage dir tempFileDir = server.createTemporaryFilename().getParentFile().getParentFile(); assertTrue(tempFileDir.setExecutable(true, false)); assertTrue(tempFileDir.setReadable(true, false)); assertTrue(tempFileDir.setWritable(false, false)); byte[] data = new byte[2000000]; rnd.nextBytes(data); // upload the file to the server via the cache exception.expect(IOException.class); exception.expectMessage("PUT operation failed: "); put(cache, jobId, data, blobType); } finally { // set writable again to make sure we can remove the directory if (tempFileDir != null) { //noinspection ResultOfMethodCallIgnored tempFileDir.setWritable(true, false); } } }
Example #23
Source File: BlockingShutdownTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testProcessShutdownBlocking() throws Exception { // this test works only on linux assumeTrue(OperatingSystem.isLinux()); final File markerFile = new File( EnvironmentInformation.getTemporaryFileDirectory(), UUID.randomUUID() + ".marker"); final BlockingShutdownProcess blockingProcess = new BlockingShutdownProcess(markerFile.getAbsolutePath(), 0, false); try { blockingProcess.startProcess(); long pid = blockingProcess.getProcessId(); assertTrue("Cannot determine process ID", pid != -1); // wait for the marker file to appear, which means the process is up properly TestJvmProcess.waitForMarkerFile(markerFile, 30000); // send it a regular kill command (SIG_TERM) Process kill = Runtime.getRuntime().exec("kill " + pid); kill.waitFor(); assertEquals("failed to send SIG_TERM to process", 0, kill.exitValue()); // minimal delay until the Java process object notices that the process is gone // this will not let the test fail predictably if the process is actually in fact going away, // but it would create frequent failures. Not ideal, but the best we can do without // severely prolonging the test Thread.sleep(50); // the process should not go away by itself assertTrue("Test broken, process shutdown blocking does not work", blockingProcess.isAlive()); } finally { blockingProcess.destroy(); //noinspection ResultOfMethodCallIgnored markerFile.delete(); } }
Example #24
Source File: BlockingShutdownTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testProcessShutdownBlocking() throws Exception { // this test works only on linux assumeTrue(OperatingSystem.isLinux()); final File markerFile = new File( EnvironmentInformation.getTemporaryFileDirectory(), UUID.randomUUID() + ".marker"); final BlockingShutdownProcess blockingProcess = new BlockingShutdownProcess(markerFile.getAbsolutePath(), 0, false); try { blockingProcess.startProcess(); long pid = blockingProcess.getProcessId(); assertTrue("Cannot determine process ID", pid != -1); // wait for the marker file to appear, which means the process is up properly TestJvmProcess.waitForMarkerFile(markerFile, 30000); // send it a regular kill command (SIG_TERM) Process kill = Runtime.getRuntime().exec("kill " + pid); kill.waitFor(); assertEquals("failed to send SIG_TERM to process", 0, kill.exitValue()); // minimal delay until the Java process object notices that the process is gone // this will not let the test fail predictably if the process is actually in fact going away, // but it would create frequent failures. Not ideal, but the best we can do without // severely prolonging the test Thread.sleep(50); // the process should not go away by itself assertTrue("Test broken, process shutdown blocking does not work", blockingProcess.isAlive()); } finally { blockingProcess.destroy(); //noinspection ResultOfMethodCallIgnored markerFile.delete(); } }
Example #25
Source File: YARNHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests that Yarn will restart a killed {@link YarnSessionClusterEntrypoint} which will then resume * a persisted {@link JobGraph}. */ @Test public void testKillYarnSessionClusterEntrypoint() throws Exception { assumeTrue( "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.", OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris()); final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor(); yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles())); final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor); try { final JobID jobId = submitJob(restClusterClient); final ApplicationId id = restClusterClient.getClusterId(); waitUntilJobIsRunning(restClusterClient, jobId); killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint()); waitForApplicationAttempt(id, 2); waitForJobTermination(restClusterClient, jobId); killApplicationAndWait(id); } finally { restClusterClient.shutdown(); } }
Example #26
Source File: YarnIntraNonHaMasterServicesTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue(!OperatingSystem.isWindows()); final File tempDir = TEMP_DIR.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path(hdfsCluster.getURI()); }
Example #27
Source File: LocalStandaloneKafkaResource.java From flink with Apache License 2.0 | 5 votes |
LocalStandaloneKafkaResource(final String kafkaVersion, @Nullable Path logBackupDirectory) { OperatingSystemRestriction.forbid( String.format("The %s relies on UNIX utils and shell scripts.", getClass().getSimpleName()), OperatingSystem.WINDOWS); this.kafkaVersion = kafkaVersion; this.logBackupDirectory = logBackupDirectory; }
Example #28
Source File: FsNegativeRunningJobsRegistryTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows()); final File tempDir = TEMP_DIR.newFolder(); Configuration hdConf = new Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path("hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/"); }
Example #29
Source File: BlobUtilsNonWritableTest.java From flink with Apache License 2.0 | 5 votes |
@Before public void before() throws IOException { assumeTrue(!OperatingSystem.isWindows()); //setWritable doesn't work on Windows. // Prepare test directory blobUtilsTestDirectory = temporaryFolder.newFolder(); assertTrue(blobUtilsTestDirectory.setExecutable(true, false)); assertTrue(blobUtilsTestDirectory.setReadable(true, false)); assertTrue(blobUtilsTestDirectory.setWritable(false, false)); }
Example #30
Source File: YarnFileStageTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void createHDFS() throws Exception { Assume.assumeTrue(!OperatingSystem.isWindows()); final File tempDir = CLASS_TEMP_DIR.newFolder(); org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); hdfsRootPath = new Path(hdfsCluster.getURI()); }