org.apache.hadoop.tools.DistCpConstants Java Examples
The following examples show how to use
org.apache.hadoop.tools.DistCpConstants.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CopyMapper.java From hadoop with Apache License 2.0 | 6 votes |
/** * Implementation of the Mapper::setup() method. This extracts the DistCp- * options specified in the Job's configuration, to set up the Job. * @param context Mapper's context. * @throws IOException On IO failure. * @throws InterruptedException If the job is interrupted. */ @Override public void setup(Context context) throws IOException, InterruptedException { conf = context.getConfiguration(); syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false); ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false); preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch. PRESERVE_STATUS.getConfigLabel())); targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); Path targetFinalPath = new Path(conf.get( DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); targetFS = targetFinalPath.getFileSystem(conf); if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) { overWrite = true; // When target is an existing file, overwrite it. } if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) { initializeSSLConf(context); } }
Example #2
Source File: TestCopyMapper.java From hadoop with Apache License 2.0 | 6 votes |
private static Configuration getConfiguration() throws IOException { Configuration configuration = getConfigurationForCluster(); final FileSystem fs = cluster.getFileSystem(); Path workPath = new Path(TARGET_PATH) .makeQualified(fs.getUri(), fs.getWorkingDirectory()); configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath.toString()); configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, workPath.toString()); configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), true); configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), "br"); return configuration; }
Example #3
Source File: CopyMapper.java From big-c with Apache License 2.0 | 6 votes |
/** * Implementation of the Mapper::setup() method. This extracts the DistCp- * options specified in the Job's configuration, to set up the Job. * @param context Mapper's context. * @throws IOException On IO failure. * @throws InterruptedException If the job is interrupted. */ @Override public void setup(Context context) throws IOException, InterruptedException { conf = context.getConfiguration(); syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false); ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false); preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch. PRESERVE_STATUS.getConfigLabel())); targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); Path targetFinalPath = new Path(conf.get( DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); targetFS = targetFinalPath.getFileSystem(conf); if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) { overWrite = true; // When target is an existing file, overwrite it. } if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) { initializeSSLConf(context); } }
Example #4
Source File: TestCopyOutputFormat.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testSetCommitDirectory() { try { Job job = Job.getInstance(new Configuration()); Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job)); job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, ""); Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job)); Path directory = new Path("/tmp/test"); CopyOutputFormat.setCommitDirectory(job, directory); Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job)); Assert.assertEquals(directory.toString(), job.getConfiguration(). get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); } catch (IOException e) { LOG.error("Exception encountered while running test", e); Assert.fail("Failed while testing for set Commit Directory"); } }
Example #5
Source File: TestCopyOutputFormat.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testSetWorkingDirectory() { try { Job job = Job.getInstance(new Configuration()); Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job)); job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, ""); Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job)); Path directory = new Path("/tmp/test"); CopyOutputFormat.setWorkingDirectory(job, directory); Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job)); Assert.assertEquals(directory.toString(), job.getConfiguration(). get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); } catch (IOException e) { LOG.error("Exception encountered while running test", e); Assert.fail("Failed while testing for set Working Directory"); } }
Example #6
Source File: TestCopyMapper.java From big-c with Apache License 2.0 | 6 votes |
private static Configuration getConfiguration() throws IOException { Configuration configuration = getConfigurationForCluster(); final FileSystem fs = cluster.getFileSystem(); Path workPath = new Path(TARGET_PATH) .makeQualified(fs.getUri(), fs.getWorkingDirectory()); configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath.toString()); configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, workPath.toString()); configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), true); configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), "br"); return configuration; }
Example #7
Source File: TestCopyOutputFormat.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testSetCommitDirectory() { try { Job job = Job.getInstance(new Configuration()); Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job)); job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, ""); Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job)); Path directory = new Path("/tmp/test"); CopyOutputFormat.setCommitDirectory(job, directory); Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job)); Assert.assertEquals(directory.toString(), job.getConfiguration(). get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); } catch (IOException e) { LOG.error("Exception encountered while running test", e); Assert.fail("Failed while testing for set Commit Directory"); } }
Example #8
Source File: TestCopyOutputFormat.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testSetWorkingDirectory() { try { Job job = Job.getInstance(new Configuration()); Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job)); job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, ""); Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job)); Path directory = new Path("/tmp/test"); CopyOutputFormat.setWorkingDirectory(job, directory); Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job)); Assert.assertEquals(directory.toString(), job.getConfiguration(). get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); } catch (IOException e) { LOG.error("Exception encountered while running test", e); Assert.fail("Failed while testing for set Working Directory"); } }
Example #9
Source File: DistCpOptionsParserTest.java From circus-train with Apache License 2.0 | 6 votes |
private void assertDefaultValues(DistCpOptions distCpOptions) { assertThat(distCpOptions, is(not(nullValue()))); assertThat(distCpOptions.preserveAttributes().hasNext(), is(false)); assertThat(distCpOptions.shouldPreserveRawXattrs(), is(false)); assertThat(distCpOptions.shouldAppend(), is(false)); assertThat(distCpOptions.shouldAtomicCommit(), is(false)); assertThat(distCpOptions.getAtomicWorkPath(), is(nullValue())); assertThat(distCpOptions.shouldBlock(), is(true)); assertThat(distCpOptions.getCopyStrategy(), is(DistCpConstants.UNIFORMSIZE)); assertThat(distCpOptions.shouldDeleteMissing(), is(false)); assertThat(distCpOptions.shouldIgnoreFailures(), is(false)); assertThat(distCpOptions.getLogPath(), is(nullValue())); assertThat(distCpOptions.getMapBandwidth(), is(DistCpConstants.DEFAULT_BANDWIDTH_MB)); assertThat(distCpOptions.getMaxMaps(), is(DistCpConstants.DEFAULT_MAPS)); assertThat(distCpOptions.shouldOverwrite(), is(false)); assertThat(distCpOptions.shouldSkipCRC(), is(false)); assertThat(distCpOptions.getSslConfigurationFile(), is(nullValue())); assertThat(distCpOptions.shouldSyncFolder(), is(false)); assertThat(distCpOptions.getTargetPathExists(), is(true)); }
Example #10
Source File: RetriableFileCopyCommand.java From big-c with Apache License 2.0 | 5 votes |
private Path getTmpFile(Path target, Mapper.Context context) { Path targetWorkPath = new Path(context.getConfiguration(). get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); Path root = target.equals(targetWorkPath)? targetWorkPath.getParent() : targetWorkPath; LOG.info("Creating temp file: " + new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString())); return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()); }
Example #11
Source File: DynamicInputFormat.java From hadoop with Apache License 2.0 | 5 votes |
private static int getMaxChunksTolerable(Configuration conf) { int maxChunksTolerable = conf.getInt( DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT); if (maxChunksTolerable <= 0) { LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE + " should be positive. Fall back to default value: " + DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT); maxChunksTolerable = DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT; } return maxChunksTolerable; }
Example #12
Source File: RetriableFileCopyCommand.java From big-c with Apache License 2.0 | 5 votes |
private static ThrottledInputStream getInputStream(Path path, Configuration conf) throws IOException { try { FileSystem fs = path.getFileSystem(conf); long bandwidthMB = conf.getInt(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, DistCpConstants.DEFAULT_BANDWIDTH_MB); FSDataInputStream in = fs.open(path); return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024); } catch (IOException e) { throw new CopyReadException(e); } }
Example #13
Source File: TestDynamicInputFormat.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testGetSplitRatio() throws Exception { Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10)); Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200)); // Tests with negative value configuration Configuration conf = new Configuration(); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, -1); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, -1); conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, -1); conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, -1); Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000, conf)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10, conf)); Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700, conf)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200, conf)); // Tests with valid configuration conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, 100); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, 30); conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, 10); conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, 53); Assert.assertEquals(53, DynamicInputFormat.getSplitRatio(3, 200, conf)); }
Example #14
Source File: UniformSizeInputFormat.java From big-c with Apache License 2.0 | 5 votes |
/** * Implementation of InputFormat::getSplits(). Returns a list of InputSplits, * such that the number of bytes to be copied for all the splits are * approximately equal. * @param context JobContext for the job. * @return The list of uniformly-distributed input-splits. * @throws IOException * @throws InterruptedException */ @Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); int numSplits = DistCpUtils.getInt(configuration, JobContext.NUM_MAPS); if (numSplits == 0) return new ArrayList<InputSplit>(); return getSplits(configuration, numSplits, DistCpUtils.getLong(configuration, DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED)); }
Example #15
Source File: TestCopyCommitter.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testAtomicCommitMissingFinal() { TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(), taskAttemptContext.getTaskAttemptID().getJobID()); Configuration conf = jobContext.getConfiguration(); String workPath = "/tmp1/" + String.valueOf(rand.nextLong()); String finalPath = "/tmp1/" + String.valueOf(rand.nextLong()); FileSystem fs = null; try { OutputCommitter committer = new CopyCommitter(null, taskAttemptContext); fs = FileSystem.get(conf); fs.mkdirs(new Path(workPath)); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true); Assert.assertTrue(fs.exists(new Path(workPath))); Assert.assertFalse(fs.exists(new Path(finalPath))); committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); //Test for idempotent commit committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); } catch (IOException e) { LOG.error("Exception encountered while testing for preserve status", e); Assert.fail("Atomic commit failure"); } finally { TestDistCpUtils.delete(fs, workPath); TestDistCpUtils.delete(fs, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false); } }
Example #16
Source File: TestCopyMapper.java From hadoop with Apache License 2.0 | 5 votes |
@Test(timeout=40000) public void testMakeDirFailure() { try { deleteState(); createSourceData(); FileSystem fs = cluster.getFileSystem(); CopyMapper copyMapper = new CopyMapper(); StubContext stubContext = new StubContext(getConfiguration(), null, 0); Mapper<Text, CopyListingFileStatus, Text, Text>.Context context = stubContext.getContext(); Configuration configuration = context.getConfiguration(); String workPath = new Path("hftp://localhost:1234/*/*/*/?/") .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(); configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); copyMapper.setup(context); copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))), new CopyListingFileStatus(fs.getFileStatus(pathList.get(0))), context); Assert.assertTrue("There should have been an exception.", false); } catch (Exception ignore) { } }
Example #17
Source File: CopyOutputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static Path getWorkingDirectory(Configuration conf) { String workingDirectory = conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH); if (workingDirectory == null || workingDirectory.isEmpty()) { return null; } else { return new Path(workingDirectory); } }
Example #18
Source File: CopyOutputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static Path getCommitDirectory(Configuration conf) { String commitDirectory = conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH); if (commitDirectory == null || commitDirectory.isEmpty()) { return null; } else { return new Path(commitDirectory); } }
Example #19
Source File: MapReduceBackupCopyJob.java From hbase with Apache License 2.0 | 5 votes |
@Override protected Path createInputFileListing(Job job) throws IOException { if (conf.get(NUMBER_OF_LEVELS_TO_PRESERVE_KEY) == null) { return super.createInputFileListing(job); } long totalBytesExpected = 0; int totalRecords = 0; Path fileListingPath = getFileListingPath(); try (SequenceFile.Writer writer = getWriter(fileListingPath)) { List<Path> srcFiles = getSourceFiles(); if (srcFiles.size() == 0) { return fileListingPath; } totalRecords = srcFiles.size(); FileSystem fs = srcFiles.get(0).getFileSystem(conf); for (Path path : srcFiles) { FileStatus fst = fs.getFileStatus(path); totalBytesExpected += fst.getLen(); Text key = getKey(path); writer.append(key, new CopyListingFileStatus(fst)); } writer.close(); // update jobs configuration Configuration cfg = job.getConfiguration(); cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, totalBytesExpected); cfg.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, fileListingPath.toString()); cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, totalRecords); } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InvocationTargetException e) { throw new IOException(e); } return fileListingPath; }
Example #20
Source File: CopyMapper.java From big-c with Apache License 2.0 | 5 votes |
/** * Initialize SSL Config if same is set in conf * * @throws IOException - If any */ private void initializeSSLConf(Context context) throws IOException { LOG.info("Initializing SSL configuration"); String workDir = conf.get(JobContext.JOB_LOCAL_DIR) + "/work"; Path[] cacheFiles = context.getLocalCacheFiles(); Configuration sslConfig = new Configuration(false); String sslConfFileName = conf.get(DistCpConstants.CONF_LABEL_SSL_CONF); Path sslClient = findCacheFile(cacheFiles, sslConfFileName); if (sslClient == null) { LOG.warn("SSL Client config file not found. Was looking for " + sslConfFileName + " in " + Arrays.toString(cacheFiles)); return; } sslConfig.addResource(sslClient); String trustStoreFile = conf.get("ssl.client.truststore.location"); Path trustStorePath = findCacheFile(cacheFiles, trustStoreFile); sslConfig.set("ssl.client.truststore.location", trustStorePath.toString()); String keyStoreFile = conf.get("ssl.client.keystore.location"); Path keyStorePath = findCacheFile(cacheFiles, keyStoreFile); sslConfig.set("ssl.client.keystore.location", keyStorePath.toString()); try { OutputStream out = new FileOutputStream(workDir + "/" + sslConfFileName); try { sslConfig.writeXml(out); } finally { out.close(); } conf.set(DistCpConstants.CONF_LABEL_SSL_KEYSTORE, sslConfFileName); } catch (IOException e) { LOG.warn("Unable to write out the ssl configuration. " + "Will fall back to default ssl-client.xml in class path, if there is one", e); } }
Example #21
Source File: DynamicInputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static int getMaxChunksTolerable(Configuration conf) { int maxChunksTolerable = conf.getInt( DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT); if (maxChunksTolerable <= 0) { LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE + " should be positive. Fall back to default value: " + DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT); maxChunksTolerable = DistCpConstants.MAX_CHUNKS_TOLERABLE_DEFAULT; } return maxChunksTolerable; }
Example #22
Source File: DynamicInputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static int getMaxChunksIdeal(Configuration conf) { int maxChunksIdeal = conf.getInt( DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT); if (maxChunksIdeal <= 0) { LOG.warn(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL + " should be positive. Fall back to default value: " + DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT); maxChunksIdeal = DistCpConstants.MAX_CHUNKS_IDEAL_DEFAULT; } return maxChunksIdeal; }
Example #23
Source File: DynamicInputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static int getMinRecordsPerChunk(Configuration conf) { int minRecordsPerChunk = conf.getInt( DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT); if (minRecordsPerChunk <= 0) { LOG.warn(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK + " should be positive. Fall back to default value: " + DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT); minRecordsPerChunk = DistCpConstants.MIN_RECORDS_PER_CHUNK_DEFAULT; } return minRecordsPerChunk; }
Example #24
Source File: DynamicInputFormat.java From big-c with Apache License 2.0 | 5 votes |
private static int getSplitRatio(Configuration conf) { int splitRatio = conf.getInt( DistCpConstants.CONF_LABEL_SPLIT_RATIO, DistCpConstants.SPLIT_RATIO_DEFAULT); if (splitRatio <= 0) { LOG.warn(DistCpConstants.CONF_LABEL_SPLIT_RATIO + " should be positive. Fall back to default value: " + DistCpConstants.SPLIT_RATIO_DEFAULT); splitRatio = DistCpConstants.SPLIT_RATIO_DEFAULT; } return splitRatio; }
Example #25
Source File: TestCopyMapper.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout=40000) public void testMakeDirFailure() { try { deleteState(); createSourceData(); FileSystem fs = cluster.getFileSystem(); CopyMapper copyMapper = new CopyMapper(); StubContext stubContext = new StubContext(getConfiguration(), null, 0); Mapper<Text, CopyListingFileStatus, Text, Text>.Context context = stubContext.getContext(); Configuration configuration = context.getConfiguration(); String workPath = new Path("hftp://localhost:1234/*/*/*/?/") .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(); configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); copyMapper.setup(context); copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))), new CopyListingFileStatus(fs.getFileStatus(pathList.get(0))), context); Assert.assertTrue("There should have been an exception.", false); } catch (Exception ignore) { } }
Example #26
Source File: TestCopyCommitter.java From big-c with Apache License 2.0 | 5 votes |
@BeforeClass public static void create() throws IOException { config = getJobForClient().getConfiguration(); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 0); cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).format(true) .build(); }
Example #27
Source File: TestCopyCommitter.java From big-c with Apache License 2.0 | 5 votes |
@Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta); } catch (IOException e) { LOG.error("Exception encountered while creating meta folder", e); Assert.fail("Unable to create meta folder"); } }
Example #28
Source File: TestCopyCommitter.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testAtomicCommitMissingFinal() { TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(), taskAttemptContext.getTaskAttemptID().getJobID()); Configuration conf = jobContext.getConfiguration(); String workPath = "/tmp1/" + String.valueOf(rand.nextLong()); String finalPath = "/tmp1/" + String.valueOf(rand.nextLong()); FileSystem fs = null; try { OutputCommitter committer = new CopyCommitter(null, taskAttemptContext); fs = FileSystem.get(conf); fs.mkdirs(new Path(workPath)); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true); Assert.assertTrue(fs.exists(new Path(workPath))); Assert.assertFalse(fs.exists(new Path(finalPath))); committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); //Test for idempotent commit committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); } catch (IOException e) { LOG.error("Exception encountered while testing for preserve status", e); Assert.fail("Atomic commit failure"); } finally { TestDistCpUtils.delete(fs, workPath); TestDistCpUtils.delete(fs, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false); } }
Example #29
Source File: TestDynamicInputFormat.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testGetSplitRatio() throws Exception { Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10)); Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200)); // Tests with negative value configuration Configuration conf = new Configuration(); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, -1); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, -1); conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, -1); conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, -1); Assert.assertEquals(1, DynamicInputFormat.getSplitRatio(1, 1000000000, conf)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(11000000, 10, conf)); Assert.assertEquals(4, DynamicInputFormat.getSplitRatio(30, 700, conf)); Assert.assertEquals(2, DynamicInputFormat.getSplitRatio(30, 200, conf)); // Tests with valid configuration conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_TOLERABLE, 100); conf.setInt(DistCpConstants.CONF_LABEL_MAX_CHUNKS_IDEAL, 30); conf.setInt(DistCpConstants.CONF_LABEL_MIN_RECORDS_PER_CHUNK, 10); conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, 53); Assert.assertEquals(53, DynamicInputFormat.getSplitRatio(3, 200, conf)); }
Example #30
Source File: CubeMigrationCrossClusterCLI.java From kylin with Apache License 2.0 | 5 votes |
/** * Set targetPathExists in both inputOptions and job config, * for the benefit of CopyCommitter */ public void setTargetPathExists(DistCpOptions inputOptions) throws IOException { Path target = inputOptions.getTargetPath(); FileSystem targetFS = target.getFileSystem(dstCluster.jobConf); boolean targetExists = targetFS.exists(target); inputOptions.setTargetPathExists(targetExists); dstCluster.jobConf.setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists); }