org.apache.hadoop.tools.DistCpConstants Java Examples

The following examples show how to use org.apache.hadoop.tools.DistCpConstants. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CopyMapper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Implementation of the Mapper::setup() method. This extracts the DistCp-
 * options specified in the Job's configuration, to set up the Job.
 * @param context Mapper's context.
 * @throws IOException On IO failure.
 * @throws InterruptedException If the job is interrupted.
 */
@Override
public void setup(Context context) throws IOException, InterruptedException {
  conf = context.getConfiguration();

  syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false);
  ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
  skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false);
  overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false);
  append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false);
  preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch.
      PRESERVE_STATUS.getConfigLabel()));

  targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  Path targetFinalPath = new Path(conf.get(
          DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  targetFS = targetFinalPath.getFileSystem(conf);

  if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) {
    overWrite = true; // When target is an existing file, overwrite it.
  }

  if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) {
    initializeSSLConf(context);
  }
}
 
Example #2
Source File: TestCopyMapper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static Configuration getConfiguration() throws IOException {
  Configuration configuration = getConfigurationForCluster();
  final FileSystem fs = cluster.getFileSystem();
  Path workPath = new Path(TARGET_PATH)
          .makeQualified(fs.getUri(), fs.getWorkingDirectory());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
          workPath.toString());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
          workPath.toString());
  configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(),
          true);
  configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
          "br");
  return configuration;
}
 
Example #3
Source File: CopyMapper.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Implementation of the Mapper::setup() method. This extracts the DistCp-
 * options specified in the Job's configuration, to set up the Job.
 * @param context Mapper's context.
 * @throws IOException On IO failure.
 * @throws InterruptedException If the job is interrupted.
 */
@Override
public void setup(Context context) throws IOException, InterruptedException {
  conf = context.getConfiguration();

  syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false);
  ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
  skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false);
  overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false);
  append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false);
  preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch.
      PRESERVE_STATUS.getConfigLabel()));

  targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  Path targetFinalPath = new Path(conf.get(
          DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  targetFS = targetFinalPath.getFileSystem(conf);

  if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) {
    overWrite = true; // When target is an existing file, overwrite it.
  }

  if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) {
    initializeSSLConf(context);
  }
}
 
Example #4
Source File: TestCopyOutputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetCommitDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setCommitDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Commit Directory");
  }
}
 
Example #5
Source File: TestCopyOutputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetWorkingDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setWorkingDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Working Directory");
  }
}
 
Example #6
Source File: TestCopyMapper.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static Configuration getConfiguration() throws IOException {
  Configuration configuration = getConfigurationForCluster();
  final FileSystem fs = cluster.getFileSystem();
  Path workPath = new Path(TARGET_PATH)
          .makeQualified(fs.getUri(), fs.getWorkingDirectory());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
          workPath.toString());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
          workPath.toString());
  configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(),
          true);
  configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
          "br");
  return configuration;
}
 
Example #7
Source File: TestCopyOutputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetCommitDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setCommitDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Commit Directory");
  }
}
 
Example #8
Source File: TestCopyOutputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetWorkingDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setWorkingDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Working Directory");
  }
}
 
Example #9
Source File: DistCpOptionsParserTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private void assertDefaultValues(DistCpOptions distCpOptions) {
  assertThat(distCpOptions, is(not(nullValue())));
  assertThat(distCpOptions.preserveAttributes().hasNext(), is(false));
  assertThat(distCpOptions.shouldPreserveRawXattrs(), is(false));
  assertThat(distCpOptions.shouldAppend(), is(false));
  assertThat(distCpOptions.shouldAtomicCommit(), is(false));
  assertThat(distCpOptions.getAtomicWorkPath(), is(nullValue()));
  assertThat(distCpOptions.shouldBlock(), is(true));
  assertThat(distCpOptions.getCopyStrategy(), is(DistCpConstants.UNIFORMSIZE));
  assertThat(distCpOptions.shouldDeleteMissing(), is(false));
  assertThat(distCpOptions.shouldIgnoreFailures(), is(false));
  assertThat(distCpOptions.getLogPath(), is(nullValue()));
  assertThat(distCpOptions.getMapBandwidth(), is(DistCpConstants.DEFAULT_BANDWIDTH_MB));
  assertThat(distCpOptions.getMaxMaps(), is(DistCpConstants.DEFAULT_MAPS));
  assertThat(distCpOptions.shouldOverwrite(), is(false));
  assertThat(distCpOptions.shouldSkipCRC(), is(false));
  assertThat(distCpOptions.getSslConfigurationFile(), is(nullValue()));
  assertThat(distCpOptions.shouldSyncFolder(), is(false));
  assertThat(distCpOptions.getTargetPathExists(), is(true));
}
 
Example #10
Source File: RetriableFileCopyCommand.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Path getTmpFile(Path target, Mapper.Context context) {
  Path targetWorkPath = new Path(context.getConfiguration().
      get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));

  Path root = target.equals(targetWorkPath)? targetWorkPath.getParent() : targetWorkPath;
  LOG.info("Creating temp file: " +
      new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()));
  return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString());
}
 
Example #11
Source File: DynamicInputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static int getMaxChunksTolerable(Configuration conf) {
  int maxChunksTolerable = conf.getInt(
      DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE,
      DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT);
  if (maxChunksTolerable <= 0) {
    LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE +
        " should be positive. Fall back to default value: "
        + DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT);
    maxChunksTolerable = DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT;
  }
  return maxChunksTolerable;
}
 
Example #12
Source File: RetriableFileCopyCommand.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static ThrottledInputStream getInputStream(Path path,
    Configuration conf) throws IOException {
  try {
    FileSystem fs = path.getFileSystem(conf);
    long bandwidthMB = conf.getInt(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
            DistCpConstants.DEFAULT_BANDWIDTH_MB);
    FSDataInputStream in = fs.open(path);
    return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024);
  }
  catch (IOException e) {
    throw new CopyReadException(e);
  }
}
 
Example #13
Source File: TestDynamicInputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSplitRatio() throws Exception {
  Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10));
  Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200));

  // Tests with negative value configuration
  Configuration conf = new Configuration();
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, -1);
  Assert.assertEquals(1,
      DynamicInputFormat.getSplitRatio(1, 1000000000, conf));
  Assert.assertEquals(2,
      DynamicInputFormat.getSplitRatio(11000000, 10, conf));
  Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700, conf));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200, conf));

  // Tests with valid configuration
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, 100);
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, 30);
  conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, 10);
  conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, 53);
  Assert.assertEquals(53, DynamicInputFormat.getSplitRatio(3, 200, conf));
}
 
Example #14
Source File: UniformSizeInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Implementation of InputFormat::getSplits(). Returns a list of InputSplits,
 * such that the number of bytes to be copied for all the splits are
 * approximately equal.
 * @param context JobContext for the job.
 * @return The list of uniformly-distributed input-splits.
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public List<InputSplit> getSplits(JobContext context)
                    throws IOException, InterruptedException {
  Configuration configuration = context.getConfiguration();
  int numSplits = DistCpUtils.getInt(configuration,
                                     JobContext.NUM_MAPS);

  if (numSplits == 0) return new ArrayList<InputSplit>();

  return getSplits(configuration, numSplits,
                   DistCpUtils.getLong(configuration,
                        DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED));
}
 
Example #15
Source File: TestCopyCommitter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAtomicCommitMissingFinal() {
  TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
  JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(),
      taskAttemptContext.getTaskAttemptID().getJobID());
  Configuration conf = jobContext.getConfiguration();

  String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
  String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
  FileSystem fs = null;
  try {
    OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
    fs = FileSystem.get(conf);
    fs.mkdirs(new Path(workPath));

    conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
    conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
    conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);

    Assert.assertTrue(fs.exists(new Path(workPath)));
    Assert.assertFalse(fs.exists(new Path(finalPath)));
    committer.commitJob(jobContext);
    Assert.assertFalse(fs.exists(new Path(workPath)));
    Assert.assertTrue(fs.exists(new Path(finalPath)));

    //Test for idempotent commit
    committer.commitJob(jobContext);
    Assert.assertFalse(fs.exists(new Path(workPath)));
    Assert.assertTrue(fs.exists(new Path(finalPath)));

  } catch (IOException e) {
    LOG.error("Exception encountered while testing for preserve status", e);
    Assert.fail("Atomic commit failure");
  } finally {
    TestDistCpUtils.delete(fs, workPath);
    TestDistCpUtils.delete(fs, finalPath);
    conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
  }
}
 
Example #16
Source File: TestCopyMapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=40000)
public void testMakeDirFailure() {
  try {
    deleteState();
    createSourceData();

    FileSystem fs = cluster.getFileSystem();
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
            = stubContext.getContext();

    Configuration configuration = context.getConfiguration();
    String workPath = new Path("hftp://localhost:1234/*/*/*/?/")
            .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
    configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
            workPath);
    copyMapper.setup(context);

    copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))),
            new CopyListingFileStatus(fs.getFileStatus(pathList.get(0))), context);

    Assert.assertTrue("There should have been an exception.", false);
  }
  catch (Exception ignore) {
  }
}
 
Example #17
Source File: CopyOutputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static Path getWorkingDirectory(Configuration conf) {
  String workingDirectory = conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH);
  if (workingDirectory == null || workingDirectory.isEmpty()) {
    return null;
  } else {
    return new Path(workingDirectory);
  }
}
 
Example #18
Source File: CopyOutputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static Path getCommitDirectory(Configuration conf) {
  String commitDirectory = conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH);
  if (commitDirectory == null || commitDirectory.isEmpty()) {
    return null;
  } else {
    return new Path(commitDirectory);
  }
}
 
Example #19
Source File: MapReduceBackupCopyJob.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected Path createInputFileListing(Job job) throws IOException {

  if (conf.get(NUMBER_OF_LEVELS_TO_PRESERVE_KEY) == null) {
    return super.createInputFileListing(job);
  }
  long totalBytesExpected = 0;
  int totalRecords = 0;
  Path fileListingPath = getFileListingPath();
  try (SequenceFile.Writer writer = getWriter(fileListingPath)) {
    List<Path> srcFiles = getSourceFiles();
    if (srcFiles.size() == 0) {
      return fileListingPath;
    }
    totalRecords = srcFiles.size();
    FileSystem fs = srcFiles.get(0).getFileSystem(conf);
    for (Path path : srcFiles) {
      FileStatus fst = fs.getFileStatus(path);
      totalBytesExpected += fst.getLen();
      Text key = getKey(path);
      writer.append(key, new CopyListingFileStatus(fst));
    }
    writer.close();

    // update jobs configuration

    Configuration cfg = job.getConfiguration();
    cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, totalBytesExpected);
    cfg.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, fileListingPath.toString());
    cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, totalRecords);
  } catch (NoSuchFieldException | SecurityException | IllegalArgumentException
      | IllegalAccessException | NoSuchMethodException | ClassNotFoundException
      | InvocationTargetException e) {
    throw new IOException(e);
  }
  return fileListingPath;
}
 
Example #20
Source File: CopyMapper.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize SSL Config if same is set in conf
 *
 * @throws IOException - If any
 */
private void initializeSSLConf(Context context) throws IOException {
  LOG.info("Initializing SSL configuration");

  String workDir = conf.get(JobContext.JOB_LOCAL_DIR) + "/work";
  Path[] cacheFiles = context.getLocalCacheFiles();

  Configuration sslConfig = new Configuration(false);
  String sslConfFileName = conf.get(DistCpConstants.CONF_LABEL_SSL_CONF);
  Path sslClient = findCacheFile(cacheFiles, sslConfFileName);
  if (sslClient == null) {
    LOG.warn("SSL Client config file not found. Was looking for " + sslConfFileName +
        " in " + Arrays.toString(cacheFiles));
    return;
  }
  sslConfig.addResource(sslClient);

  String trustStoreFile = conf.get("ssl.client.truststore.location");
  Path trustStorePath = findCacheFile(cacheFiles, trustStoreFile);
  sslConfig.set("ssl.client.truststore.location", trustStorePath.toString());

  String keyStoreFile = conf.get("ssl.client.keystore.location");
  Path keyStorePath = findCacheFile(cacheFiles, keyStoreFile);
  sslConfig.set("ssl.client.keystore.location", keyStorePath.toString());

  try {
    OutputStream out = new FileOutputStream(workDir + "/" + sslConfFileName);
    try {
      sslConfig.writeXml(out);
    } finally {
      out.close();
    }
    conf.set(DistCpConstants.CONF_LABEL_SSL_KEYSTORE, sslConfFileName);
  } catch (IOException e) {
    LOG.warn("Unable to write out the ssl configuration. " +
        "Will fall back to default ssl-client.xml in class path, if there is one", e);
  }
}
 
Example #21
Source File: DynamicInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static int getMaxChunksTolerable(Configuration conf) {
  int maxChunksTolerable = conf.getInt(
      DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE,
      DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT);
  if (maxChunksTolerable <= 0) {
    LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE +
        " should be positive. Fall back to default value: "
        + DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT);
    maxChunksTolerable = DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT;
  }
  return maxChunksTolerable;
}
 
Example #22
Source File: DynamicInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static int getMaxChunksIdeal(Configuration conf) {
  int maxChunksIdeal = conf.getInt(
      DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL,
      DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT);
  if (maxChunksIdeal <= 0) {
    LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL +
        " should be positive. Fall back to default value: "
        + DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT);
    maxChunksIdeal = DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT;
  }
  return maxChunksIdeal;
}
 
Example #23
Source File: DynamicInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static int getMinRecordsPerChunk(Configuration conf) {
  int minRecordsPerChunk = conf.getInt(
      DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK,
      DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT);
  if (minRecordsPerChunk <= 0) {
    LOG.warn(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK +
        " should be positive. Fall back to default value: "
        + DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT);
    minRecordsPerChunk = DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT;
  }
  return minRecordsPerChunk;
}
 
Example #24
Source File: DynamicInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static int getSplitRatio(Configuration conf) {
  int splitRatio = conf.getInt(
      DistCpConstants.CONF_LABEL_SPLIT_RATIO,
      DistCpConstants.SPLIT_RATIO_DEFAULT);
  if (splitRatio <= 0) {
    LOG.warn(DistCpConstants.CONF_LABEL_SPLIT_RATIO +
        " should be positive. Fall back to default value: "
        + DistCpConstants.SPLIT_RATIO_DEFAULT);
    splitRatio = DistCpConstants.SPLIT_RATIO_DEFAULT;
  }
  return splitRatio;
}
 
Example #25
Source File: TestCopyMapper.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=40000)
public void testMakeDirFailure() {
  try {
    deleteState();
    createSourceData();

    FileSystem fs = cluster.getFileSystem();
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
            = stubContext.getContext();

    Configuration configuration = context.getConfiguration();
    String workPath = new Path("hftp://localhost:1234/*/*/*/?/")
            .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
    configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
            workPath);
    copyMapper.setup(context);

    copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))),
            new CopyListingFileStatus(fs.getFileStatus(pathList.get(0))), context);

    Assert.assertTrue("There should have been an exception.", false);
  }
  catch (Exception ignore) {
  }
}
 
Example #26
Source File: TestCopyCommitter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void create() throws IOException {
  config = getJobForClient().getConfiguration();
  config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 0);
  cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).format(true)
                    .build();
}
 
Example #27
Source File: TestCopyCommitter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void createMetaFolder() {
  config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
  Path meta = new Path("/meta");
  try {
    cluster.getFileSystem().mkdirs(meta);
  } catch (IOException e) {
    LOG.error("Exception encountered while creating meta folder", e);
    Assert.fail("Unable to create meta folder");
  }
}
 
Example #28
Source File: TestCopyCommitter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAtomicCommitMissingFinal() {
  TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
  JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(),
      taskAttemptContext.getTaskAttemptID().getJobID());
  Configuration conf = jobContext.getConfiguration();

  String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
  String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
  FileSystem fs = null;
  try {
    OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
    fs = FileSystem.get(conf);
    fs.mkdirs(new Path(workPath));

    conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
    conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
    conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);

    Assert.assertTrue(fs.exists(new Path(workPath)));
    Assert.assertFalse(fs.exists(new Path(finalPath)));
    committer.commitJob(jobContext);
    Assert.assertFalse(fs.exists(new Path(workPath)));
    Assert.assertTrue(fs.exists(new Path(finalPath)));

    //Test for idempotent commit
    committer.commitJob(jobContext);
    Assert.assertFalse(fs.exists(new Path(workPath)));
    Assert.assertTrue(fs.exists(new Path(finalPath)));

  } catch (IOException e) {
    LOG.error("Exception encountered while testing for preserve status", e);
    Assert.fail("Atomic commit failure");
  } finally {
    TestDistCpUtils.delete(fs, workPath);
    TestDistCpUtils.delete(fs, finalPath);
    conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
  }
}
 
Example #29
Source File: TestDynamicInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSplitRatio() throws Exception {
  Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10));
  Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200));

  // Tests with negative value configuration
  Configuration conf = new Configuration();
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, -1);
  conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, -1);
  Assert.assertEquals(1,
      DynamicInputFormat.getSplitRatio(1, 1000000000, conf));
  Assert.assertEquals(2,
      DynamicInputFormat.getSplitRatio(11000000, 10, conf));
  Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700, conf));
  Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200, conf));

  // Tests with valid configuration
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, 100);
  conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, 30);
  conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, 10);
  conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, 53);
  Assert.assertEquals(53, DynamicInputFormat.getSplitRatio(3, 200, conf));
}
 
Example #30
Source File: CubeMigrationCrossClusterCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Set targetPathExists in both inputOptions and job config,
 * for the benefit of CopyCommitter
 */
public void setTargetPathExists(DistCpOptions inputOptions) throws IOException {
    Path target = inputOptions.getTargetPath();
    FileSystem targetFS = target.getFileSystem(dstCluster.jobConf);
    boolean targetExists = targetFS.exists(target);
    inputOptions.setTargetPathExists(targetExists);
    dstCluster.jobConf.setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists);
}