org.apache.hadoop.tools.DistCp Java Examples

The following examples show how to use org.apache.hadoop.tools.DistCp. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BaseUploaderTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
public void init(int numPartition) throws Exception {
  this.numPartition = numPartition;
  sourceFiles = HFileGenerator.generateHFiles(fs, conf, tempFolder,
      options.loadOptions.getPartitioner(), numPartition, numPartition * 1000);
  blockSize = 0;
  for (Path path : sourceFiles) {
    long fileSize = new File(path.toString()).length();
    if (fileSize > blockSize) {
      blockSize = fileSize;
    }
  }
  zkManager = mock(ZooKeeperManager.class);
  distCp = mock(DistCp.class);
  job = mock(Job.class);
  when(zkManager.getClusterInfo()).thenReturn(new ClusterInfo(NAME_NODE, REPLICA_FACTOR));
  when(distCp.execute()).thenReturn(job);
  when(job.waitForCompletion(anyBoolean())).then(new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
      Thread.sleep(1000);
      return true;
    }
  });
  doNothing().when(zkManager).lockFileSet(anyString(), any(FileSetInfo.class));
  doNothing().when(zkManager).unlockFileSet(anyString());
}
 
Example #2
Source File: BaseUploaderTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
@Override
protected DistCp getDistCp(Configuration conf, DistCpOptions options) {
  assertEquals(Constants.MAPRED_MAP_MAX_ATTEMPTS,
      Integer.parseInt(conf.get("mapred.map.max.attempts")));
  assertEquals(Constants.CHECKSUM_BYTES,
      Integer.parseInt(conf.get("io.bytes.per.checksum")));
  long blockSizeExpected = blockSize;
  if (blockSizeExpected % Constants.CHECKSUM_BYTES != 0) {
    blockSizeExpected = (blockSize / Constants.CHECKSUM_BYTES + 1) * Constants.CHECKSUM_BYTES;
  }
  assertEquals(blockSizeExpected, Long.parseLong(conf.get("dfs.block.size")));
  assertEquals(REPLICA_FACTOR, Integer.parseInt(conf.get("dfs.replication")));
  assertEquals(sourceFiles, options.getSourcePaths());
  assertTrue(options.shouldSkipCRC());
  assertTrue(options.shouldSyncFolder());
  assertTrue(options.getTargetPath().toString().startsWith("hdfs://" + NAME_NODE + HDFS_DIR));
  if (numPartition == 1) {
    assertTrue(options.getTargetPath().toString()
        .endsWith(TerrapinUtil.formatPartitionName(0)));
  }
  return distCp;
}
 
Example #3
Source File: TestCopyFiles.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 
Example #4
Source File: TestCopyFiles.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 
Example #5
Source File: CubeMigrationCrossClusterCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected void copyHDFSPath(String srcDir, Configuration srcConf, String dstDir, Configuration dstConf)
        throws Exception {
    logger.info("start to copy hdfs directory from {} to {}", srcDir, dstDir);
    DistCpOptions distCpOptions = OptionsParser.parse(new String[] { srcDir, dstDir });
    distCpOptions.preserve(DistCpOptions.FileAttribute.BLOCKSIZE);
    distCpOptions.setBlocking(true);
    setTargetPathExists(distCpOptions);
    DistCp distCp = new DistCp(getConfOfDistCp(), distCpOptions);
    distCp.execute();
    logger.info("copied hdfs directory from {} to {}", srcDir, dstDir);
}
 
Example #6
Source File: MapReduceBackupCopyJob.java    From hbase with Apache License 2.0 5 votes vote down vote up
private List<Path> getSourceFiles() throws NoSuchFieldException, SecurityException,
    IllegalArgumentException, IllegalAccessException, NoSuchMethodException,
    ClassNotFoundException, InvocationTargetException, IOException {
  Field options = null;
  try {
    options = DistCp.class.getDeclaredField("inputOptions");
  } catch (NoSuchFieldException | SecurityException e) {
    options = DistCp.class.getDeclaredField("context");
  }
  options.setAccessible(true);
  return getSourcePaths(options);
}
 
Example #7
Source File: TestCopyFiles.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
  MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
  ToolRunner.run(new DistCp(new Configuration()),
                         new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
  assertTrue("Source and destination directories do not match.",
             checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 
Example #8
Source File: TestCopyFiles.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
  MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
  ToolRunner.run(new DistCp(new Configuration()),
                         new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
  assertTrue("Source and destination directories do not match.",
             checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 
Example #9
Source File: DistCpCopier.java    From circus-train with Apache License 2.0 4 votes vote down vote up
@Override
public Job exec(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options).execute();
}
 
Example #10
Source File: BaseUploader.java    From terrapin with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected DistCp getDistCp(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options);
}
 
Example #11
Source File: TestCopyFiles.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public void testCopySingleFile() throws Exception {
  FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
  Path root = new Path(TEST_ROOT_DIR+"/srcdat");
  try {    
    MyFile[] files = {createFile(root, fs)};
    //copy a dir with a single file
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
    
    //copy a single file
    String fname = files[0].getName();
    Path p = new Path(root, fname);
    FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));     
    //copy single file to existing dir
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
    MyFile[] files2 = {createFile(root, fs, 0)};
    String sname = files2[0].getName();
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
    updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
    //copy single file to existing dir w/ dst name conflict
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
  }
  finally {
    deldir(fs, TEST_ROOT_DIR+"/destdat");
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    deldir(fs, TEST_ROOT_DIR+"/srcdat");
  }
}
 
Example #12
Source File: TestCopyFiles.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
public void testCopySingleFile() throws Exception {
  FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
  Path root = new Path(TEST_ROOT_DIR+"/srcdat");
  try {    
    MyFile[] files = {createFile(root, fs)};
    //copy a dir with a single file
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
    
    //copy a single file
    String fname = files[0].getName();
    Path p = new Path(root, fname);
    FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));     
    //copy single file to existing dir
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
    MyFile[] files2 = {createFile(root, fs, 0)};
    String sname = files2[0].getName();
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
    updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
    //copy single file to existing dir w/ dst name conflict
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
  }
  finally {
    deldir(fs, TEST_ROOT_DIR+"/destdat");
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    deldir(fs, TEST_ROOT_DIR+"/srcdat");
  }
}
 
Example #13
Source File: DistCpTestUtils.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Runs distcp from src to dst, preserving XAttrs. Asserts the
 * expected exit code.
 *
 * @param exitCode expected exit code
 * @param src distcp src path
 * @param dst distcp destination
 * @param options distcp command line options
 * @param conf Configuration to use
 * @throws Exception if there is any error
 */
public static void assertRunDistCp(int exitCode, String src, String dst,
    String options, Configuration conf)
    throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = options == null ?
      new String[] { src, dst } :
      new String[] { options, src, dst };
  assertEquals(exitCode,
      ToolRunner.run(conf, distCp, optsArr));
}
 
Example #14
Source File: DistCpTestUtils.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Runs distcp from src to dst, preserving XAttrs. Asserts the
 * expected exit code.
 *
 * @param exitCode expected exit code
 * @param src distcp src path
 * @param dst distcp destination
 * @param options distcp command line options
 * @param conf Configuration to use
 * @throws Exception if there is any error
 */
public static void assertRunDistCp(int exitCode, String src, String dst,
    String options, Configuration conf)
    throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = options == null ?
      new String[] { src, dst } :
      new String[] { options, src, dst };
  assertEquals(exitCode,
      ToolRunner.run(conf, distCp, optsArr));
}