Java Code Examples for org.apache.hadoop.mapred.MiniMRCluster#shutdown()
The following examples show how to use
org.apache.hadoop.mapred.MiniMRCluster#shutdown() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestUlimit.java From RDFS with Apache License 2.0 | 6 votes |
/** * This tests the setting of memory limit for streaming processes. * This will launch a streaming app which will allocate 10MB memory. * First, program is launched with sufficient memory. And test expects * it to succeed. Then program is launched with insufficient memory and * is expected to be a failure. */ public void testCommandLine() { if (StreamUtil.isCygwin()) { return; } try { final int numSlaves = 2; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, numSlaves, true, null); fs = dfs.getFileSystem(); mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1); writeInputFile(fs, inputPath); map = StreamUtil.makeJavaCommand(UlimitApp.class, new String[]{}); runProgram(SET_MEMORY_LIMIT); fs.delete(outputPath, true); assertFalse("output not cleaned up", fs.exists(outputPath)); mr.waitUntilIdle(); } catch(IOException e) { fail(e.toString()); } finally { mr.shutdown(); dfs.shutdown(); } }
Example 2
Source File: TestUlimit.java From hadoop-gpu with Apache License 2.0 | 6 votes |
/** * This tests the setting of memory limit for streaming processes. * This will launch a streaming app which will allocate 10MB memory. * First, program is launched with sufficient memory. And test expects * it to succeed. Then program is launched with insufficient memory and * is expected to be a failure. */ public void testCommandLine() { if (StreamUtil.isCygwin()) { return; } try { final int numSlaves = 2; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, numSlaves, true, null); fs = dfs.getFileSystem(); mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1); writeInputFile(fs, inputPath); map = StreamUtil.makeJavaCommand(UlimitApp.class, new String[]{}); runProgram(SET_MEMORY_LIMIT); fs.delete(outputPath, true); assertFalse("output not cleaned up", fs.exists(outputPath)); mr.waitUntilIdle(); } catch(IOException e) { fail(e.toString()); } finally { mr.shutdown(); dfs.shutdown(); } }
Example 3
Source File: TestRaidShell.java From RDFS with Apache License 2.0 | 5 votes |
/** * Test distRaid command * @throws Exception */ public void testDistRaid() throws Exception { LOG.info("TestDist started."); // create a dfs and map-reduce cluster mySetup(3, -1); MiniMRCluster mr = new MiniMRCluster(4, namenode, 3); String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); conf.set("mapred.job.tracker", jobTrackerName); try { // Create files to be raided TestRaidNode.createTestFiles(fileSys, RAID_SRC_PATH, "/raid" + RAID_SRC_PATH, 1, 3, (short)3); String subDir = RAID_SRC_PATH + "/subdir"; TestRaidNode.createTestFiles( fileSys, subDir, "/raid" + subDir, 1, 3, (short)3); // Create RaidShell and raid the files. RaidShell shell = new RaidShell(conf); String[] args = new String[3]; args[0] = "-distRaid"; args[1] = RAID_POLICY_NAME; args[2] = RAID_SRC_PATH; assertEquals(0, ToolRunner.run(shell, args)); // Check files are raided checkIfFileRaided(new Path(RAID_SRC_PATH, "file0")); checkIfFileRaided(new Path(subDir, "file0")); } finally { mr.shutdown(); myTearDown(); } }
Example 4
Source File: TestDistributionPolicy.java From RDFS with Apache License 2.0 | 4 votes |
protected void setUp() throws Exception { super.setUp(); try { dfsCluster = new MiniDFSCluster(conf, numDataNodes, true, (String[]) null); fs = dfsCluster.getFileSystem(); if (fs.exists(inputPath)) { fs.delete(inputPath); } fs.copyFromLocalFile(localInputPath, inputPath); if (fs.exists(updatePath)) { fs.delete(updatePath); } fs.copyFromLocalFile(localUpdatePath, updatePath); if (fs.exists(outputPath)) { // do not create, mapred will create fs.delete(outputPath); } if (fs.exists(indexPath)) { fs.delete(indexPath); } mrCluster = new MiniMRCluster(numTaskTrackers, fs.getUri().toString(), 1); } catch (IOException e) { if (dfsCluster != null) { dfsCluster.shutdown(); dfsCluster = null; } if (fs != null) { fs.close(); fs = null; } if (mrCluster != null) { mrCluster.shutdown(); mrCluster = null; } throw e; } }
Example 5
Source File: TestIndexUpdater.java From RDFS with Apache License 2.0 | 4 votes |
protected void setUp() throws Exception { super.setUp(); try { dfsCluster = new MiniDFSCluster(conf, numDataNodes, true, (String[]) null); fs = dfsCluster.getFileSystem(); if (fs.exists(inputPath)) { fs.delete(inputPath); } fs.copyFromLocalFile(localInputPath, inputPath); if (fs.exists(outputPath)) { // do not create, mapred will create fs.delete(outputPath); } if (fs.exists(indexPath)) { fs.delete(indexPath); } mrCluster = new MiniMRCluster(numTaskTrackers, fs.getUri().toString(), 1); } catch (IOException e) { if (dfsCluster != null) { dfsCluster.shutdown(); dfsCluster = null; } if (fs != null) { fs.close(); fs = null; } if (mrCluster != null) { mrCluster.shutdown(); mrCluster = null; } throw e; } }
Example 6
Source File: TestDistributionPolicy.java From hadoop-gpu with Apache License 2.0 | 4 votes |
protected void setUp() throws Exception { super.setUp(); try { dfsCluster = new MiniDFSCluster(conf, numDataNodes, true, (String[]) null); fs = dfsCluster.getFileSystem(); if (fs.exists(inputPath)) { fs.delete(inputPath); } fs.copyFromLocalFile(localInputPath, inputPath); if (fs.exists(updatePath)) { fs.delete(updatePath); } fs.copyFromLocalFile(localUpdatePath, updatePath); if (fs.exists(outputPath)) { // do not create, mapred will create fs.delete(outputPath); } if (fs.exists(indexPath)) { fs.delete(indexPath); } mrCluster = new MiniMRCluster(numTaskTrackers, fs.getUri().toString(), 1); } catch (IOException e) { if (dfsCluster != null) { dfsCluster.shutdown(); dfsCluster = null; } if (fs != null) { fs.close(); fs = null; } if (mrCluster != null) { mrCluster.shutdown(); mrCluster = null; } throw e; } }
Example 7
Source File: TestIndexUpdater.java From hadoop-gpu with Apache License 2.0 | 4 votes |
protected void setUp() throws Exception { super.setUp(); try { dfsCluster = new MiniDFSCluster(conf, numDataNodes, true, (String[]) null); fs = dfsCluster.getFileSystem(); if (fs.exists(inputPath)) { fs.delete(inputPath); } fs.copyFromLocalFile(localInputPath, inputPath); if (fs.exists(outputPath)) { // do not create, mapred will create fs.delete(outputPath); } if (fs.exists(indexPath)) { fs.delete(indexPath); } mrCluster = new MiniMRCluster(numTaskTrackers, fs.getUri().toString(), 1); } catch (IOException e) { if (dfsCluster != null) { dfsCluster.shutdown(); dfsCluster = null; } if (fs != null) { fs.close(); fs = null; } if (mrCluster != null) { mrCluster.shutdown(); mrCluster = null; } throw e; } }