Java Code Examples for org.apache.hadoop.tools.DistCp

The following examples show how to use org.apache.hadoop.tools.DistCp. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: terrapin   Source File: BaseUploaderTest.java    License: Apache License 2.0 6 votes vote down vote up
public void init(int numPartition) throws Exception {
  this.numPartition = numPartition;
  sourceFiles = HFileGenerator.generateHFiles(fs, conf, tempFolder,
      options.loadOptions.getPartitioner(), numPartition, numPartition * 1000);
  blockSize = 0;
  for (Path path : sourceFiles) {
    long fileSize = new File(path.toString()).length();
    if (fileSize > blockSize) {
      blockSize = fileSize;
    }
  }
  zkManager = mock(ZooKeeperManager.class);
  distCp = mock(DistCp.class);
  job = mock(Job.class);
  when(zkManager.getClusterInfo()).thenReturn(new ClusterInfo(NAME_NODE, REPLICA_FACTOR));
  when(distCp.execute()).thenReturn(job);
  when(job.waitForCompletion(anyBoolean())).then(new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
      Thread.sleep(1000);
      return true;
    }
  });
  doNothing().when(zkManager).lockFileSet(anyString(), any(FileSetInfo.class));
  doNothing().when(zkManager).unlockFileSet(anyString());
}
 
Example 2
Source Project: terrapin   Source File: BaseUploaderTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected DistCp getDistCp(Configuration conf, DistCpOptions options) {
  assertEquals(Constants.MAPRED_MAP_MAX_ATTEMPTS,
      Integer.parseInt(conf.get("mapred.map.max.attempts")));
  assertEquals(Constants.CHECKSUM_BYTES,
      Integer.parseInt(conf.get("io.bytes.per.checksum")));
  long blockSizeExpected = blockSize;
  if (blockSizeExpected % Constants.CHECKSUM_BYTES != 0) {
    blockSizeExpected = (blockSize / Constants.CHECKSUM_BYTES + 1) * Constants.CHECKSUM_BYTES;
  }
  assertEquals(blockSizeExpected, Long.parseLong(conf.get("dfs.block.size")));
  assertEquals(REPLICA_FACTOR, Integer.parseInt(conf.get("dfs.replication")));
  assertEquals(sourceFiles, options.getSourcePaths());
  assertTrue(options.shouldSkipCRC());
  assertTrue(options.shouldSyncFolder());
  assertTrue(options.getTargetPath().toString().startsWith("hdfs://" + NAME_NODE + HDFS_DIR));
  if (numPartition == 1) {
    assertTrue(options.getTargetPath().toString()
        .endsWith(TerrapinUtil.formatPartitionName(0)));
  }
  return distCp;
}
 
Example 3
Source Project: RDFS   Source File: TestCopyFiles.java    License: Apache License 2.0 6 votes vote down vote up
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 
Example 4
Source Project: hadoop-gpu   Source File: TestCopyFiles.java    License: Apache License 2.0 6 votes vote down vote up
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 
Example 5
Source Project: kylin   Source File: CubeMigrationCrossClusterCLI.java    License: Apache License 2.0 5 votes vote down vote up
protected void copyHDFSPath(String srcDir, Configuration srcConf, String dstDir, Configuration dstConf)
        throws Exception {
    logger.info("start to copy hdfs directory from {} to {}", srcDir, dstDir);
    DistCpOptions distCpOptions = OptionsParser.parse(new String[] { srcDir, dstDir });
    distCpOptions.preserve(DistCpOptions.FileAttribute.BLOCKSIZE);
    distCpOptions.setBlocking(true);
    setTargetPathExists(distCpOptions);
    DistCp distCp = new DistCp(getConfOfDistCp(), distCpOptions);
    distCp.execute();
    logger.info("copied hdfs directory from {} to {}", srcDir, dstDir);
}
 
Example 6
Source Project: hbase   Source File: MapReduceBackupCopyJob.java    License: Apache License 2.0 5 votes vote down vote up
private List<Path> getSourceFiles() throws NoSuchFieldException, SecurityException,
    IllegalArgumentException, IllegalAccessException, NoSuchMethodException,
    ClassNotFoundException, InvocationTargetException, IOException {
  Field options = null;
  try {
    options = DistCp.class.getDeclaredField("inputOptions");
  } catch (NoSuchFieldException | SecurityException e) {
    options = DistCp.class.getDeclaredField("context");
  }
  options.setAccessible(true);
  return getSourcePaths(options);
}
 
Example 7
Source Project: RDFS   Source File: TestCopyFiles.java    License: Apache License 2.0 5 votes vote down vote up
/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
  MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
  ToolRunner.run(new DistCp(new Configuration()),
                         new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
  assertTrue("Source and destination directories do not match.",
             checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 
Example 8
Source Project: hadoop-gpu   Source File: TestCopyFiles.java    License: Apache License 2.0 5 votes vote down vote up
/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
  MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
  ToolRunner.run(new DistCp(new Configuration()),
                         new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
  assertTrue("Source and destination directories do not match.",
             checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 
Example 9
Source Project: circus-train   Source File: DistCpCopier.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Job exec(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options).execute();
}
 
Example 10
Source Project: terrapin   Source File: BaseUploader.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected DistCp getDistCp(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options);
}
 
Example 11
Source Project: RDFS   Source File: TestCopyFiles.java    License: Apache License 2.0 4 votes vote down vote up
public void testCopySingleFile() throws Exception {
  FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
  Path root = new Path(TEST_ROOT_DIR+"/srcdat");
  try {    
    MyFile[] files = {createFile(root, fs)};
    //copy a dir with a single file
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
    
    //copy a single file
    String fname = files[0].getName();
    Path p = new Path(root, fname);
    FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));     
    //copy single file to existing dir
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
    MyFile[] files2 = {createFile(root, fs, 0)};
    String sname = files2[0].getName();
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
    updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
    //copy single file to existing dir w/ dst name conflict
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
  }
  finally {
    deldir(fs, TEST_ROOT_DIR+"/destdat");
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    deldir(fs, TEST_ROOT_DIR+"/srcdat");
  }
}
 
Example 12
Source Project: hadoop-gpu   Source File: TestCopyFiles.java    License: Apache License 2.0 4 votes vote down vote up
public void testCopySingleFile() throws Exception {
  FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
  Path root = new Path(TEST_ROOT_DIR+"/srcdat");
  try {    
    MyFile[] files = {createFile(root, fs)};
    //copy a dir with a single file
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
    
    //copy a single file
    String fname = files[0].getName();
    Path p = new Path(root, fname);
    FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));     
    //copy single file to existing dir
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
    MyFile[] files2 = {createFile(root, fs, 0)};
    String sname = files2[0].getName();
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
    updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
    //copy single file to existing dir w/ dst name conflict
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"-update",
                      "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
                      "file:///"+TEST_ROOT_DIR+"/dest2/"});
    assertTrue("Source and destination directories do not match.",
        checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
  }
  finally {
    deldir(fs, TEST_ROOT_DIR+"/destdat");
    deldir(fs, TEST_ROOT_DIR+"/dest2");
    deldir(fs, TEST_ROOT_DIR+"/srcdat");
  }
}
 
Example 13
Source Project: hadoop   Source File: DistCpTestUtils.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Runs distcp from src to dst, preserving XAttrs. Asserts the
 * expected exit code.
 *
 * @param exitCode expected exit code
 * @param src distcp src path
 * @param dst distcp destination
 * @param options distcp command line options
 * @param conf Configuration to use
 * @throws Exception if there is any error
 */
public static void assertRunDistCp(int exitCode, String src, String dst,
    String options, Configuration conf)
    throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = options == null ?
      new String[] { src, dst } :
      new String[] { options, src, dst };
  assertEquals(exitCode,
      ToolRunner.run(conf, distCp, optsArr));
}
 
Example 14
Source Project: big-c   Source File: DistCpTestUtils.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Runs distcp from src to dst, preserving XAttrs. Asserts the
 * expected exit code.
 *
 * @param exitCode expected exit code
 * @param src distcp src path
 * @param dst distcp destination
 * @param options distcp command line options
 * @param conf Configuration to use
 * @throws Exception if there is any error
 */
public static void assertRunDistCp(int exitCode, String src, String dst,
    String options, Configuration conf)
    throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = options == null ?
      new String[] { src, dst } :
      new String[] { options, src, dst };
  assertEquals(exitCode,
      ToolRunner.run(conf, distCp, optsArr));
}