Java Code Examples for org.apache.hadoop.filecache.DistributedCache#addCacheArchive()

The following examples show how to use org.apache.hadoop.filecache.DistributedCache#addCacheArchive() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SolrOutputFormat.java From examples with Apache License 2.0

6 votes

public static void addSolrConfToDistributedCache(Job job, File solrHomeZip)
    throws IOException {
  // Make a reasonably unique name for the zip file in the distributed cache
  // to avoid collisions if multiple jobs are running.
  String hdfsZipName = UUID.randomUUID().toString() + '.'
      + ZIP_FILE_BASE_NAME;
  Configuration jobConf = job.getConfiguration();
  jobConf.set(ZIP_NAME, hdfsZipName);

  Path zipPath = new Path("/tmp", getZipName(jobConf));
  FileSystem fs = FileSystem.get(jobConf);
  fs.copyFromLocalFile(new Path(solrHomeZip.toString()), zipPath);
  final URI baseZipUrl = fs.getUri().resolve(
      zipPath.toString() + '#' + getZipName(jobConf));

  DistributedCache.addCacheArchive(baseZipUrl, jobConf);
  LOG.debug("Set Solr distributed cache: {}", Arrays.asList(job.getCacheArchives()));
  LOG.debug("Set zipPath: {}", zipPath);
  // Actually send the path for the configuration zip file
  jobConf.set(SETUP_OK, zipPath.toString());
}

Example 2

Source File: JobControlCompiler.java From spork with Apache License 2.0

5 votes

private static void addToDistributedCache(URI uri, Configuration conf) {
    if (DISTRIBUTED_CACHE_ARCHIVE_MATCHER.reset(uri.toString()).find()) {
        DistributedCache.addCacheArchive(uri, conf);
    } else {
        DistributedCache.addCacheFile(uri, conf);
    }
}

Example 3

Source File: TestMRWithDistributedCache.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 1000)
public void testDeprecatedFunctions() throws Exception {
  DistributedCache.addLocalArchives(conf, "Test Local Archives 1");
  Assert.assertEquals("Test Local Archives 1",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 1",
      DistributedCache.getLocalCacheArchives(conf)[0].getName());
  DistributedCache.addLocalArchives(conf, "Test Local Archives 2");
  Assert.assertEquals("Test Local Archives 1,Test Local Archives 2",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(2,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 2",
      DistributedCache.getLocalCacheArchives(conf)[1].getName());
  DistributedCache.setLocalArchives(conf, "Test Local Archives 3");
  Assert.assertEquals("Test Local Archives 3",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 3",
      DistributedCache.getLocalCacheArchives(conf)[0].getName());

  DistributedCache.addLocalFiles(conf, "Test Local Files 1");
  Assert.assertEquals("Test Local Files 1",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 1",
      DistributedCache.getLocalCacheFiles(conf)[0].getName());
  DistributedCache.addLocalFiles(conf, "Test Local Files 2");
  Assert.assertEquals("Test Local Files 1,Test Local Files 2",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(2,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 2",
      DistributedCache.getLocalCacheFiles(conf)[1].getName());
  DistributedCache.setLocalFiles(conf, "Test Local Files 3");
  Assert.assertEquals("Test Local Files 3",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 3",
      DistributedCache.getLocalCacheFiles(conf)[0].getName());

  DistributedCache.setArchiveTimestamps(conf, "1234567890");
  Assert.assertEquals(1234567890,
      conf.getLong(DistributedCache.CACHE_ARCHIVES_TIMESTAMPS, 0));
  Assert.assertEquals(1,
      DistributedCache.getArchiveTimestamps(conf).length);
  Assert.assertEquals(1234567890,
      DistributedCache.getArchiveTimestamps(conf)[0]);
  DistributedCache.setFileTimestamps(conf, "1234567890");
  Assert.assertEquals(1234567890,
      conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0));
  Assert.assertEquals(1,
      DistributedCache.getFileTimestamps(conf).length);
  Assert.assertEquals(1234567890,
      DistributedCache.getFileTimestamps(conf)[0]);

  DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"),
      new File("Test Work Dir"));
  Assert.assertNull(conf.get(DistributedCache.CACHE_SYMLINK));
  Assert.assertTrue(DistributedCache.getSymlink(conf));

  Assert.assertTrue(symlinkFile.createNewFile());
  FileStatus fileStatus =
      DistributedCache.getFileStatus(conf, symlinkFile.toURI());
  Assert.assertNotNull(fileStatus);
  Assert.assertEquals(fileStatus.getModificationTime(),
      DistributedCache.getTimestamp(conf, symlinkFile.toURI()));
  Assert.assertTrue(symlinkFile.delete());

  DistributedCache.addCacheArchive(symlinkFile.toURI(), conf);
  Assert.assertEquals(symlinkFile.toURI().toString(),
      conf.get(DistributedCache.CACHE_ARCHIVES));
  Assert.assertEquals(1, DistributedCache.getCacheArchives(conf).length);
  Assert.assertEquals(symlinkFile.toURI(),
      DistributedCache.getCacheArchives(conf)[0]);

  DistributedCache.addCacheFile(symlinkFile.toURI(), conf);
  Assert.assertEquals(symlinkFile.toURI().toString(),
      conf.get(DistributedCache.CACHE_FILES));
  Assert.assertEquals(1, DistributedCache.getCacheFiles(conf).length);
  Assert.assertEquals(symlinkFile.toURI(),
      DistributedCache.getCacheFiles(conf)[0]);
}

Example 4

Source File: TestMRWithDistributedCache.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 1000)
public void testDeprecatedFunctions() throws Exception {
  DistributedCache.addLocalArchives(conf, "Test Local Archives 1");
  Assert.assertEquals("Test Local Archives 1",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 1",
      DistributedCache.getLocalCacheArchives(conf)[0].getName());
  DistributedCache.addLocalArchives(conf, "Test Local Archives 2");
  Assert.assertEquals("Test Local Archives 1,Test Local Archives 2",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(2,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 2",
      DistributedCache.getLocalCacheArchives(conf)[1].getName());
  DistributedCache.setLocalArchives(conf, "Test Local Archives 3");
  Assert.assertEquals("Test Local Archives 3",
      conf.get(DistributedCache.CACHE_LOCALARCHIVES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheArchives(conf).length);
  Assert.assertEquals("Test Local Archives 3",
      DistributedCache.getLocalCacheArchives(conf)[0].getName());

  DistributedCache.addLocalFiles(conf, "Test Local Files 1");
  Assert.assertEquals("Test Local Files 1",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 1",
      DistributedCache.getLocalCacheFiles(conf)[0].getName());
  DistributedCache.addLocalFiles(conf, "Test Local Files 2");
  Assert.assertEquals("Test Local Files 1,Test Local Files 2",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(2,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 2",
      DistributedCache.getLocalCacheFiles(conf)[1].getName());
  DistributedCache.setLocalFiles(conf, "Test Local Files 3");
  Assert.assertEquals("Test Local Files 3",
      conf.get(DistributedCache.CACHE_LOCALFILES));
  Assert.assertEquals(1,
      DistributedCache.getLocalCacheFiles(conf).length);
  Assert.assertEquals("Test Local Files 3",
      DistributedCache.getLocalCacheFiles(conf)[0].getName());

  DistributedCache.setArchiveTimestamps(conf, "1234567890");
  Assert.assertEquals(1234567890,
      conf.getLong(DistributedCache.CACHE_ARCHIVES_TIMESTAMPS, 0));
  Assert.assertEquals(1,
      DistributedCache.getArchiveTimestamps(conf).length);
  Assert.assertEquals(1234567890,
      DistributedCache.getArchiveTimestamps(conf)[0]);
  DistributedCache.setFileTimestamps(conf, "1234567890");
  Assert.assertEquals(1234567890,
      conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0));
  Assert.assertEquals(1,
      DistributedCache.getFileTimestamps(conf).length);
  Assert.assertEquals(1234567890,
      DistributedCache.getFileTimestamps(conf)[0]);

  DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"),
      new File("Test Work Dir"));
  Assert.assertNull(conf.get(DistributedCache.CACHE_SYMLINK));
  Assert.assertTrue(DistributedCache.getSymlink(conf));

  Assert.assertTrue(symlinkFile.createNewFile());
  FileStatus fileStatus =
      DistributedCache.getFileStatus(conf, symlinkFile.toURI());
  Assert.assertNotNull(fileStatus);
  Assert.assertEquals(fileStatus.getModificationTime(),
      DistributedCache.getTimestamp(conf, symlinkFile.toURI()));
  Assert.assertTrue(symlinkFile.delete());

  DistributedCache.addCacheArchive(symlinkFile.toURI(), conf);
  Assert.assertEquals(symlinkFile.toURI().toString(),
      conf.get(DistributedCache.CACHE_ARCHIVES));
  Assert.assertEquals(1, DistributedCache.getCacheArchives(conf).length);
  Assert.assertEquals(symlinkFile.toURI(),
      DistributedCache.getCacheArchives(conf)[0]);

  DistributedCache.addCacheFile(symlinkFile.toURI(), conf);
  Assert.assertEquals(symlinkFile.toURI().toString(),
      conf.get(DistributedCache.CACHE_FILES));
  Assert.assertEquals(1, DistributedCache.getCacheFiles(conf).length);
  Assert.assertEquals(symlinkFile.toURI(),
      DistributedCache.getCacheFiles(conf)[0]);
}