org.apache.hadoop.tools.DistCp Java Examples
The following examples show how to use
org.apache.hadoop.tools.DistCp.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BaseUploaderTest.java From terrapin with Apache License 2.0 | 6 votes |
public void init(int numPartition) throws Exception { this.numPartition = numPartition; sourceFiles = HFileGenerator.generateHFiles(fs, conf, tempFolder, options.loadOptions.getPartitioner(), numPartition, numPartition * 1000); blockSize = 0; for (Path path : sourceFiles) { long fileSize = new File(path.toString()).length(); if (fileSize > blockSize) { blockSize = fileSize; } } zkManager = mock(ZooKeeperManager.class); distCp = mock(DistCp.class); job = mock(Job.class); when(zkManager.getClusterInfo()).thenReturn(new ClusterInfo(NAME_NODE, REPLICA_FACTOR)); when(distCp.execute()).thenReturn(job); when(job.waitForCompletion(anyBoolean())).then(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Thread.sleep(1000); return true; } }); doNothing().when(zkManager).lockFileSet(anyString(), any(FileSetInfo.class)); doNothing().when(zkManager).unlockFileSet(anyString()); }
Example #2
Source File: BaseUploaderTest.java From terrapin with Apache License 2.0 | 6 votes |
@Override protected DistCp getDistCp(Configuration conf, DistCpOptions options) { assertEquals(Constants.MAPRED_MAP_MAX_ATTEMPTS, Integer.parseInt(conf.get("mapred.map.max.attempts"))); assertEquals(Constants.CHECKSUM_BYTES, Integer.parseInt(conf.get("io.bytes.per.checksum"))); long blockSizeExpected = blockSize; if (blockSizeExpected % Constants.CHECKSUM_BYTES != 0) { blockSizeExpected = (blockSize / Constants.CHECKSUM_BYTES + 1) * Constants.CHECKSUM_BYTES; } assertEquals(blockSizeExpected, Long.parseLong(conf.get("dfs.block.size"))); assertEquals(REPLICA_FACTOR, Integer.parseInt(conf.get("dfs.replication"))); assertEquals(sourceFiles, options.getSourcePaths()); assertTrue(options.shouldSkipCRC()); assertTrue(options.shouldSyncFolder()); assertTrue(options.getTargetPath().toString().startsWith("hdfs://" + NAME_NODE + HDFS_DIR)); if (numPartition == 1) { assertTrue(options.getTargetPath().toString() .endsWith(TerrapinUtil.formatPartitionName(0))); } return distCp; }
Example #3
Source File: TestCopyFiles.java From RDFS with Apache License 2.0 | 6 votes |
public void testCopyDuplication() throws Exception { final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration()); try { MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat"); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/src2/srcdat"}); assertTrue("Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files)); assertEquals(DistCp.DuplicationException.ERROR_CODE, ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/src2/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat",})); } finally { deldir(localfs, TEST_ROOT_DIR+"/destdat"); deldir(localfs, TEST_ROOT_DIR+"/srcdat"); deldir(localfs, TEST_ROOT_DIR+"/src2"); } }
Example #4
Source File: TestCopyFiles.java From hadoop-gpu with Apache License 2.0 | 6 votes |
public void testCopyDuplication() throws Exception { final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration()); try { MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat"); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/src2/srcdat"}); assertTrue("Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files)); assertEquals(DistCp.DuplicationException.ERROR_CODE, ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/src2/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat",})); } finally { deldir(localfs, TEST_ROOT_DIR+"/destdat"); deldir(localfs, TEST_ROOT_DIR+"/srcdat"); deldir(localfs, TEST_ROOT_DIR+"/src2"); } }
Example #5
Source File: CubeMigrationCrossClusterCLI.java From kylin with Apache License 2.0 | 5 votes |
protected void copyHDFSPath(String srcDir, Configuration srcConf, String dstDir, Configuration dstConf) throws Exception { logger.info("start to copy hdfs directory from {} to {}", srcDir, dstDir); DistCpOptions distCpOptions = OptionsParser.parse(new String[] { srcDir, dstDir }); distCpOptions.preserve(DistCpOptions.FileAttribute.BLOCKSIZE); distCpOptions.setBlocking(true); setTargetPathExists(distCpOptions); DistCp distCp = new DistCp(getConfOfDistCp(), distCpOptions); distCp.execute(); logger.info("copied hdfs directory from {} to {}", srcDir, dstDir); }
Example #6
Source File: MapReduceBackupCopyJob.java From hbase with Apache License 2.0 | 5 votes |
private List<Path> getSourceFiles() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, NoSuchMethodException, ClassNotFoundException, InvocationTargetException, IOException { Field options = null; try { options = DistCp.class.getDeclaredField("inputOptions"); } catch (NoSuchFieldException | SecurityException e) { options = DistCp.class.getDeclaredField("context"); } options.setAccessible(true); return getSourcePaths(options); }
Example #7
Source File: TestCopyFiles.java From RDFS with Apache License 2.0 | 5 votes |
/** copy files from local file system to local file system */ public void testCopyFromLocalToLocal() throws Exception { Configuration conf = new Configuration(); FileSystem localfs = FileSystem.get(LOCAL_FS, conf); MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat"); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat"}); assertTrue("Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files)); deldir(localfs, TEST_ROOT_DIR+"/destdat"); deldir(localfs, TEST_ROOT_DIR+"/srcdat"); }
Example #8
Source File: TestCopyFiles.java From hadoop-gpu with Apache License 2.0 | 5 votes |
/** copy files from local file system to local file system */ public void testCopyFromLocalToLocal() throws Exception { Configuration conf = new Configuration(); FileSystem localfs = FileSystem.get(LOCAL_FS, conf); MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat"); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat"}); assertTrue("Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files)); deldir(localfs, TEST_ROOT_DIR+"/destdat"); deldir(localfs, TEST_ROOT_DIR+"/srcdat"); }
Example #9
Source File: DistCpCopier.java From circus-train with Apache License 2.0 | 4 votes |
@Override public Job exec(Configuration conf, DistCpOptions options) throws Exception { return new DistCp(conf, options).execute(); }
Example #10
Source File: BaseUploader.java From terrapin with Apache License 2.0 | 4 votes |
@VisibleForTesting protected DistCp getDistCp(Configuration conf, DistCpOptions options) throws Exception { return new DistCp(conf, options); }
Example #11
Source File: TestCopyFiles.java From RDFS with Apache License 2.0 | 4 votes |
public void testCopySingleFile() throws Exception { FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration()); Path root = new Path(TEST_ROOT_DIR+"/srcdat"); try { MyFile[] files = {createFile(root, fs)}; //copy a dir with a single file ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/destdat", files)); //copy a single file String fname = files[0].getName(); Path p = new Path(root, fname); FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p)); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname, "file:///"+TEST_ROOT_DIR+"/dest2/"+fname}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files)); //copy single file to existing dir deldir(fs, TEST_ROOT_DIR+"/dest2"); fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2")); MyFile[] files2 = {createFile(root, fs, 0)}; String sname = files2[0].getName(); ToolRunner.run(new DistCp(new Configuration()), new String[] {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, "file:///"+TEST_ROOT_DIR+"/dest2/"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1); //copy single file to existing dir w/ dst name conflict ToolRunner.run(new DistCp(new Configuration()), new String[] {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, "file:///"+TEST_ROOT_DIR+"/dest2/"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); } finally { deldir(fs, TEST_ROOT_DIR+"/destdat"); deldir(fs, TEST_ROOT_DIR+"/dest2"); deldir(fs, TEST_ROOT_DIR+"/srcdat"); } }
Example #12
Source File: TestCopyFiles.java From hadoop-gpu with Apache License 2.0 | 4 votes |
public void testCopySingleFile() throws Exception { FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration()); Path root = new Path(TEST_ROOT_DIR+"/srcdat"); try { MyFile[] files = {createFile(root, fs)}; //copy a dir with a single file ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/destdat", files)); //copy a single file String fname = files[0].getName(); Path p = new Path(root, fname); FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p)); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname, "file:///"+TEST_ROOT_DIR+"/dest2/"+fname}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files)); //copy single file to existing dir deldir(fs, TEST_ROOT_DIR+"/dest2"); fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2")); MyFile[] files2 = {createFile(root, fs, 0)}; String sname = files2[0].getName(); ToolRunner.run(new DistCp(new Configuration()), new String[] {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, "file:///"+TEST_ROOT_DIR+"/dest2/"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1); //copy single file to existing dir w/ dst name conflict ToolRunner.run(new DistCp(new Configuration()), new String[] {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, "file:///"+TEST_ROOT_DIR+"/dest2/"}); assertTrue("Source and destination directories do not match.", checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); } finally { deldir(fs, TEST_ROOT_DIR+"/destdat"); deldir(fs, TEST_ROOT_DIR+"/dest2"); deldir(fs, TEST_ROOT_DIR+"/srcdat"); } }
Example #13
Source File: DistCpTestUtils.java From hadoop with Apache License 2.0 | 3 votes |
/** * Runs distcp from src to dst, preserving XAttrs. Asserts the * expected exit code. * * @param exitCode expected exit code * @param src distcp src path * @param dst distcp destination * @param options distcp command line options * @param conf Configuration to use * @throws Exception if there is any error */ public static void assertRunDistCp(int exitCode, String src, String dst, String options, Configuration conf) throws Exception { DistCp distCp = new DistCp(conf, null); String[] optsArr = options == null ? new String[] { src, dst } : new String[] { options, src, dst }; assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); }
Example #14
Source File: DistCpTestUtils.java From big-c with Apache License 2.0 | 3 votes |
/** * Runs distcp from src to dst, preserving XAttrs. Asserts the * expected exit code. * * @param exitCode expected exit code * @param src distcp src path * @param dst distcp destination * @param options distcp command line options * @param conf Configuration to use * @throws Exception if there is any error */ public static void assertRunDistCp(int exitCode, String src, String dst, String options, Configuration conf) throws Exception { DistCp distCp = new DistCp(conf, null); String[] optsArr = options == null ? new String[] { src, dst } : new String[] { options, src, dst }; assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); }