org.apache.hadoop.util.ClassUtil Java Examples

The following examples show how to use org.apache.hadoop.util.ClassUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HddsVersionInfo.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
  System.out.println("Using HDDS " + HDDS_VERSION_INFO.getVersion());
  System.out.println(
      "Source code repository " + HDDS_VERSION_INFO.getUrl() + " -r " +
          HDDS_VERSION_INFO.getRevision());
  System.out.println("Compiled by " + HDDS_VERSION_INFO.getUser() + " on "
      + HDDS_VERSION_INFO.getDate());
  System.out.println(
      "Compiled with protoc " + HDDS_VERSION_INFO.getProtocVersion());
  System.out.println(
      "From source with checksum " + HDDS_VERSION_INFO.getSrcChecksum());
  if (LOG.isDebugEnabled()) {
    LOG.debug("This command was run using " +
        ClassUtil.findContainingJar(HddsVersionInfo.class));
  }
}
 
Example #2
Source File: TestMRCJCJobConf.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Given a path with a jar, make a classloader with that jar on the
 * classpath, and check that findContainingJar can correctly
 * identify the path of the jar.
 */
private void testJarAtPath(String path) throws Exception {
  File jar = new File(path).getAbsoluteFile();
  assertTrue(jar.exists());

  URL urls[] = new URL[] {
    jar.toURI().toURL()
  };

  ClassLoader cl = new URLClassLoader(urls);
  Class clazz = Class.forName(CLASSNAME, true, cl);
  assertNotNull(clazz);

  String containingJar = ClassUtil.findContainingJar(clazz);
  assertEquals(jar.getAbsolutePath(), containingJar);
}
 
Example #3
Source File: HadoopUtils.java    From mrgeo with Apache License 2.0 6 votes vote down vote up
public static String getJar(Configuration conf, Class clazz) throws IOException
{

  if (isLocal(conf))
  {
    String jar = ClassUtil.findContainingJar(clazz);

    if (jar != null)
    {
      conf.set("mapreduce.job.jar", jar);
    }
  }

  return DependencyLoader.getMasterJar(clazz);
  //setJar(job.getConfiguration());
}
 
Example #4
Source File: HadoopUtils.java    From mrgeo with Apache License 2.0 6 votes vote down vote up
public static void setJar(Job job, Class clazz) throws IOException
{
  Configuration conf = job.getConfiguration();

  if (isLocal(conf))
  {
    String jar = ClassUtil.findContainingJar(clazz);

    if (jar != null)
    {
      conf.set("mapreduce.job.jar", jar);
    }
  }
  else
  {
    DependencyLoader.addDependencies(job, clazz);
    DataProviderFactory.addDependencies(conf);
  }
}
 
Example #5
Source File: TestMRCJCJobConf.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Given a path with a jar, make a classloader with that jar on the
 * classpath, and check that findContainingJar can correctly
 * identify the path of the jar.
 */
private void testJarAtPath(String path) throws Exception {
  File jar = new File(path).getAbsoluteFile();
  assertTrue(jar.exists());

  URL urls[] = new URL[] {
    jar.toURI().toURL()
  };

  ClassLoader cl = new URLClassLoader(urls);
  Class clazz = Class.forName(CLASSNAME, true, cl);
  assertNotNull(clazz);

  String containingJar = ClassUtil.findContainingJar(clazz);
  assertEquals(jar.getAbsolutePath(), containingJar);
}
 
Example #6
Source File: OzoneVersionInfo.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
  System.out.println(
      "                  //////////////                 \n" +
      "               ////////////////////              \n" +
      "            ////////     ////////////////        \n" +
      "           //////      ////////////////          \n" +
      "          /////      ////////////////  /         \n" +
      "         /////            ////////   ///         \n" +
      "         ////           ////////    /////        \n" +
      "        /////         ////////////////           \n" +
      "        /////       ////////////////   //        \n" +
      "         ////     ///////////////   /////        \n" +
      "         /////  ///////////////     ////         \n" +
      "          /////       //////      /////          \n" +
      "           //////   //////       /////           \n" +
      "             ///////////     ////////            \n" +
      "               //////  ////////////              \n" +
      "               ///   //////////                  \n" +
          "              /    " + OZONE_VERSION_INFO.getVersion() + "("
          + OZONE_VERSION_INFO.getRelease() + ")\n");
  System.out.println(
      "Source code repository " + OZONE_VERSION_INFO.getUrl() + " -r " +
          OZONE_VERSION_INFO.getRevision());
  System.out.println("Compiled by " + OZONE_VERSION_INFO.getUser() + " on "
      + OZONE_VERSION_INFO.getDate());
  System.out.println(
      "Compiled with protoc " + OZONE_VERSION_INFO.getProtocVersion());
  System.out.println(
      "From source with checksum " + OZONE_VERSION_INFO.getSrcChecksum()
          + "\n");
  LOG.debug("This command was run using " +
      ClassUtil.findContainingJar(OzoneVersionInfo.class));
  HddsVersionInfo.main(args);
}
 
Example #7
Source File: VersionInfo.java    From tez with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
  if (args.length != 1) {
    System.err.println("Invalid no. of args. Usage: VersionInfo <component-name>");
    System.exit(-1);
  }

  VersionInfo versionInfo = new VersionInfo(args[0]);
  System.out.println("VersionInfo: " + versionInfo.toString());
  System.out.println("This command was run using " +
      ClassUtil.findContainingJar(VersionInfo.class));
}
 
Example #8
Source File: EmbeddedGobblin.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
/**
 * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value
 * will appear first in the classpath. Default priority is 0.
 */
public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) {
  String jar = ClassUtil.findContainingJar(klazz);
  if (jar == null) {
    log.warn(String.format("Could not find jar for class %s. This is normal in test runs.", klazz));
    return this;
  }
  return distributeJarWithPriority(jar, priority);
}
 
Example #9
Source File: VersionInfo.java    From tajo with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
  LOG.debug("version: "+ getVersion());
  System.out.println("Tajo " + getVersion());
  System.out.println("Git " + getUrl() + " -r " + getRevision());
  System.out.println("Compiled by " + getUser() + " on " + getDate());
  System.out.println("Compiled with protoc " + getProtocVersion());
  System.out.println("From source with checksum " + getSrcChecksum());
  System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class));
}
 
Example #10
Source File: JobConf.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Set the job's jar file by finding an example class location.
 * 
 * @param cls the example class.
 */
public void setJarByClass(Class cls) {
  String jar = ClassUtil.findContainingJar(cls);
  if (jar != null) {
    setJar(jar);
  }   
}
 
Example #11
Source File: JobConf.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Set the job's jar file by finding an example class location.
 * 
 * @param cls the example class.
 */
public void setJarByClass(Class cls) {
  String jar = ClassUtil.findContainingJar(cls);
  if (jar != null) {
    setJar(jar);
  }   
}
 
Example #12
Source File: TestMRJobs.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testSleepJobInternal(boolean useRemoteJar) throws Exception {
  LOG.info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar);

  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
             + " not found. Not running test.");
    return;
  }

  Configuration sleepConf = new Configuration(mrCluster.getConfig());
  // set master address to local to test that local mode applied iff framework == local
  sleepConf.set(MRConfig.MASTER_ADDRESS, "local");	
  
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(sleepConf);
 
  // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each:
  Job job = sleepJob.createJob(3, numSleepReducers, 10000, 1, 5000, 1);

  job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
  if (useRemoteJar) {
    final Path localJar = new Path(
        ClassUtil.findContainingJar(SleepJob.class));
    ConfigUtil.addLink(job.getConfiguration(), "/jobjars",
        localFs.makeQualified(localJar.getParent()).toUri());
    job.setJar("viewfs:///jobjars/" + localJar.getName());
  } else {
    job.setJarByClass(SleepJob.class);
  }
  job.setMaxMapAttempts(1); // speed up failures
  job.submit();
  String trackingUrl = job.getTrackingURL();
  String jobId = job.getJobID().toString();
  boolean succeeded = job.waitForCompletion(true);
  Assert.assertTrue(succeeded);
  Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
  Assert.assertTrue("Tracking URL was " + trackingUrl +
                    " but didn't Match Job ID " + jobId ,
        trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
  verifySleepJobCounters(job);
  verifyTaskProgress(job);
  
  // TODO later:  add explicit "isUber()" checks of some sort (extend
  // JobStatus?)--compare against MRJobConfig.JOB_UBERTASK_ENABLE value
}
 
Example #13
Source File: TestMRJobs.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void testSleepJobInternal(boolean useRemoteJar) throws Exception {
  LOG.info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar);

  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
             + " not found. Not running test.");
    return;
  }

  Configuration sleepConf = new Configuration(mrCluster.getConfig());
  // set master address to local to test that local mode applied iff framework == local
  sleepConf.set(MRConfig.MASTER_ADDRESS, "local");	
  
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(sleepConf);
 
  // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each:
  Job job = sleepJob.createJob(3, numSleepReducers, 10000, 1, 5000, 1);

  job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
  if (useRemoteJar) {
    final Path localJar = new Path(
        ClassUtil.findContainingJar(SleepJob.class));
    ConfigUtil.addLink(job.getConfiguration(), "/jobjars",
        localFs.makeQualified(localJar.getParent()).toUri());
    job.setJar("viewfs:///jobjars/" + localJar.getName());
  } else {
    job.setJarByClass(SleepJob.class);
  }
  job.setMaxMapAttempts(1); // speed up failures
  job.submit();
  String trackingUrl = job.getTrackingURL();
  String jobId = job.getJobID().toString();
  boolean succeeded = job.waitForCompletion(true);
  Assert.assertTrue(succeeded);
  Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
  Assert.assertTrue("Tracking URL was " + trackingUrl +
                    " but didn't Match Job ID " + jobId ,
        trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
  verifySleepJobCounters(job);
  verifyTaskProgress(job);
  
  // TODO later:  add explicit "isUber()" checks of some sort (extend
  // JobStatus?)--compare against MRJobConfig.JOB_UBERTASK_ENABLE value
}
 
Example #14
Source File: DependencyLoader.java    From mrgeo with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("squid:S1166") // Exception caught and handled
private static Set<Dependency> loadDependenciesByReflection(Class<?> clazz) throws IOException
{
  String jar = ClassUtil.findContainingJar(clazz);

  if (jar != null)
  {
    return loadDependenciesFromJar(jar);
  }
  else
  {
    // the properties may have been added on the classpath, lets see if we can find it...
    Set<Dependency> deps = null;

    // set up a resource scanner
    Reflections reflections = new Reflections(new ConfigurationBuilder()
        .setUrls(ClasspathHelper.forPackage(ClassUtils.getPackageName(DependencyLoader.class)))
        .setScanners(new ResourcesScanner()));

    Set<String> resources = reflections.getResources(Pattern.compile(".*dependencies\\.properties"));
    for (String resource : resources)
    {
      log.debug("Loading dependency properties from: /" + resource);

      InputStream is = DependencyLoader.class.getResourceAsStream("/" + resource);

      try
      {
        Set<Dependency> d = readDependencies(is);
        is.close();

        if (deps == null)
        {
          deps = d;
        }
        else
        {
          deps.addAll(d);
        }
      }
      finally
      {
        if (is != null)
        {
          try
          {
            is.close();
          }
          catch (IOException ignored)
          {
          }
        }
      }
    }

    return deps;
  }
}
 
Example #15
Source File: LaunchCluster.java    From TensorFlowOnYARN with Apache License 2.0 4 votes vote down vote up
public LaunchCluster(Configuration conf, YarnClient yarnClient, CommandLine cliParser) {
  this.conf = conf;
  this.yarnClient = yarnClient;
  appName = cliParser.getOptionValue(
      Constants.OPT_TF_APP_NAME, Constants.DEFAULT_APP_NAME);

  amMemory = Integer.parseInt(cliParser.getOptionValue(
      Constants.OPT_TF_APP_MASTER_MEMORY, Constants.DEFAULT_APP_MASTER_MEMORY));
  amVCores = Integer.parseInt(cliParser.getOptionValue(
      Constants.OPT_TF_APP_MASTER_VCORES, Constants.DEFAULT_APP_MASTER_VCORES));
  amQueue = cliParser.getOptionValue(
      Constants.OPT_TF_APP_MASTER_QUEUE, Constants.DEFAULT_APP_MASTER_QUEUE);
  containerMemory = Integer.parseInt(cliParser.getOptionValue(
      Constants.OPT_TF_CONTAINER_MEMORY, Constants.DEFAULT_CONTAINER_MEMORY));
  containerVCores = Integer.parseInt(cliParser.getOptionValue(
      Constants.OPT_TF_CONTAINER_VCORES, Constants.DEFAULT_CONTAINER_VCORES));

  if (cliParser.hasOption(Constants.OPT_TF_JAR)) {
    tfJar = cliParser.getOptionValue(Constants.OPT_TF_JAR);
  } else {
    tfJar = ClassUtil.findContainingJar(getClass());
  }

  if (cliParser.hasOption(Constants.OPT_TF_LIB)) {
    tfLib = cliParser.getOptionValue(Constants.OPT_TF_LIB);
  } else {
    tfLib = Utils.getParentDir(tfJar) + File.separator + Constants.TF_LIB_NAME;
  }

  workerNum = Integer.parseInt(
      cliParser.getOptionValue(Constants.OPT_TF_WORKER_NUM, Constants.DEFAULT_TF_WORKER_NUM));

  if (workerNum <= 0) {
    throw new IllegalArgumentException(
        "Illegal number of TensorFlow worker task specified: " + workerNum);
  }

  psNum = Integer.parseInt(
      cliParser.getOptionValue(Constants.OPT_TF_PS_NUM, Constants.DEFAULT_TF_PS_NUM));

  if (psNum < 0) {
    throw new IllegalArgumentException(
        "Illegal number of TensorFlow ps task specified: " + psNum);
  }
}
 
Example #16
Source File: JobConf.java    From big-c with Apache License 2.0 2 votes vote down vote up
/** 
 * Find a jar that contains a class of the same name, if any.
 * It will return a jar file, even if that is not the first thing
 * on the class path that has a class with the same name.
 * 
 * @param my_class the class to find.
 * @return a jar file that contains the class, or null.
 */
public static String findContainingJar(Class my_class) {
  return ClassUtil.findContainingJar(my_class);
}
 
Example #17
Source File: JobConf.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/** 
 * Find a jar that contains a class of the same name, if any.
 * It will return a jar file, even if that is not the first thing
 * on the class path that has a class with the same name.
 * 
 * @param my_class the class to find.
 * @return a jar file that contains the class, or null.
 */
public static String findContainingJar(Class my_class) {
  return ClassUtil.findContainingJar(my_class);
}