Java Code Examples for org.apache.hadoop.conf.Configuration#getClassByName()

The following examples show how to use org.apache.hadoop.conf.Configuration#getClassByName() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: rubix   File: CachingFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initializeClusterManager(Configuration conf, ClusterType clusterType)
    throws ClusterManagerInitilizationException
{
  if (clusterManager != null) {
    return;
  }

  String clusterManagerClassName = CacheConfig.getClusterManagerClass(conf, clusterType);
  log.info("Initializing cluster manager : " + clusterManagerClassName);

  try {
    Class clusterManagerClass = conf.getClassByName(clusterManagerClassName);
    Constructor constructor = clusterManagerClass.getConstructor();
    ClusterManager manager = (ClusterManager) constructor.newInstance();

    manager.initialize(conf);
    setClusterManager(manager);
  }
  catch (ClassNotFoundException | NoSuchMethodException | InstantiationException |
          IllegalAccessException | InvocationTargetException ex) {
    String errorMessage = String.format("Not able to initialize ClusterManager class : {0} ",
        clusterManagerClassName);
    log.error(errorMessage, ex);
    throw new ClusterManagerInitilizationException(errorMessage, ex);
  }
}
 
Example 2
/**
 * Returns a {@link List} containing <em>all</em> of the topic-group-{@link Mapper} combinations added via
 * {@link #addTopic(Job, String, String, Class)}.
 * 
 * @param conf
 *            the conf for this job.
 * @return all of the configured {@link TopicConf}s
 */
@SuppressWarnings("unchecked")
public static List<TopicConf> getTopics(final Configuration conf) {
    final List<TopicConf> result = Lists.newArrayList();
    for (final String topicConf : conf.get(TOPICS_CONF).split(";")) {
        final String[] topicConfTokens = topicConf.split(",");
        final String topic = topicConfTokens[0];
        final String group = topicConfTokens[1];
        final Class<? extends Mapper> mapper;
        try {
            mapper = (Class<? extends Mapper>) conf.getClassByName(topicConfTokens[2]);
        } catch (final ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        result.add(new TopicConf(topic, group, mapper));
    }
    return result;
}
 
Example 3
Source Project: parquet-mr   File: ConfigurationUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static Class<?> getClassFromConfig(Configuration configuration, String configName, Class<?> assignableFrom) {
  final String className = configuration.get(configName);
  if (className == null) {
    return null;
  }
  
  try {
    final Class<?> foundClass = configuration.getClassByName(className);	
    if (!assignableFrom.isAssignableFrom(foundClass)) {
      throw new BadConfigurationException("class " + className + " set in job conf at "
              + configName + " is not a subclass of " + assignableFrom.getCanonicalName());
    }
    return foundClass;
  } catch (ClassNotFoundException e) {
    throw new BadConfigurationException("could not instantiate class " + className + " set in job conf at " + configName, e);
  }
}
 
Example 4
Source Project: RDFS   File: StreamUtil.java    License: Apache License 2.0 5 votes vote down vote up
/** It may seem strange to silently switch behaviour when a String
 * is not a classname; the reason is simplified Usage:<pre>
 * -mapper [classname | program ]
 * instead of the explicit Usage:
 * [-mapper program | -javamapper classname], -mapper and -javamapper are mutually exclusive.
 * (repeat for -reducer, -combiner) </pre>
 */
public static Class goodClassOrNull(Configuration conf, String className, String defaultPackage) {
  if (className.indexOf('.') == -1 && defaultPackage != null) {
    className = defaultPackage + "." + className;
  }
  Class clazz = null;
  try {
    clazz = conf.getClassByName(className);
  } catch (ClassNotFoundException cnf) {
  }
  return clazz;
}
 
Example 5
Source Project: hadoop-gpu   File: SerializationFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
        ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serilization class not found: " +
        StringUtils.stringifyException(e));
  }
}
 
Example 6
Source Project: hadoop   File: NetUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the socket factory corresponding to the given proxy URI. If the
 * given proxy URI corresponds to an absence of configuration parameter,
 * returns null. If the URI is malformed raises an exception.
 * 
 * @param propValue the property which is the class name of the
 *        SocketFactory to instantiate; assumed non null and non empty.
 * @return a socket factory as defined in the property value.
 */
public static SocketFactory getSocketFactoryFromProperty(
    Configuration conf, String propValue) {

  try {
    Class<?> theClass = conf.getClassByName(propValue);
    return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);

  } catch (ClassNotFoundException cnfe) {
    throw new RuntimeException("Socket Factory class not found: " + cnfe);
  }
}
 
Example 7
Source Project: hadoop   File: Server.java    License: Apache License 2.0 5 votes vote down vote up
static Class<?> getProtocolClass(String protocolName, Configuration conf) 
throws ClassNotFoundException {
  Class<?> protocol = PROTOCOL_CACHE.get(protocolName);
  if (protocol == null) {
    protocol = conf.getClassByName(protocolName);
    PROTOCOL_CACHE.put(protocolName, protocol);
  }
  return protocol;
}
 
Example 8
Source Project: hadoop   File: CompressionCodecFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the list of codecs discovered via a Java ServiceLoader, or
 * listed in the configuration. Codecs specified in configuration come
 * later in the returned list, and are considered to override those
 * from the ServiceLoader.
 * @param conf the configuration to look in
 * @return a list of the {@link CompressionCodec} classes
 */
public static List<Class<? extends CompressionCodec>> getCodecClasses(
    Configuration conf) {
  List<Class<? extends CompressionCodec>> result
    = new ArrayList<Class<? extends CompressionCodec>>();
  // Add codec classes discovered via service loading
  synchronized (CODEC_PROVIDERS) {
    // CODEC_PROVIDERS is a lazy collection. Synchronize so it is
    // thread-safe. See HADOOP-8406.
    for (CompressionCodec codec : CODEC_PROVIDERS) {
      result.add(codec.getClass());
    }
  }
  // Add codec classes from configuration
  String codecsString = conf.get(
      CommonConfigurationKeys.IO_COMPRESSION_CODECS_KEY);
  if (codecsString != null) {
    StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
    while (codecSplit.hasMoreElements()) {
      String codecSubstring = codecSplit.nextToken().trim();
      if (codecSubstring.length() != 0) {
        try {
          Class<?> cls = conf.getClassByName(codecSubstring);
          if (!CompressionCodec.class.isAssignableFrom(cls)) {
            throw new IllegalArgumentException("Class " + codecSubstring +
                                               " is not a CompressionCodec");
          }
          result.add(cls.asSubclass(CompressionCodec.class));
        } catch (ClassNotFoundException ex) {
          throw new IllegalArgumentException("Compression codec " + 
                                             codecSubstring + " not found.",
                                             ex);
        }
      }
    }
  }
  return result;
}
 
Example 9
Source Project: hadoop   File: WritableName.java    License: Apache License 2.0 5 votes vote down vote up
/** Return the class for a name.  Default is {@link Class#forName(String)}.*/
public static synchronized Class<?> getClass(String name, Configuration conf
                                          ) throws IOException {
  Class<?> writableClass = NAME_TO_CLASS.get(name);
  if (writableClass != null)
    return writableClass.asSubclass(Writable.class);
  try {
    return conf.getClassByName(name);
  } catch (ClassNotFoundException e) {
    IOException newE = new IOException("WritableName can't load class: " + name);
    newE.initCause(e);
    throw newE;
  }
}
 
Example 10
Source Project: hadoop   File: SerializationFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
    ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serialization class not found: ", e);
  }
}
 
Example 11
Source Project: hadoop-gpu   File: WritableName.java    License: Apache License 2.0 5 votes vote down vote up
/** Return the class for a name.  Default is {@link Class#forName(String)}.*/
public static synchronized Class<?> getClass(String name, Configuration conf
                                          ) throws IOException {
  Class<?> writableClass = NAME_TO_CLASS.get(name);
  if (writableClass != null)
    return writableClass.asSubclass(Writable.class);
  try {
    return conf.getClassByName(name);
  } catch (ClassNotFoundException e) {
    IOException newE = new IOException("WritableName can't load class: " + name);
    newE.initCause(e);
    throw newE;
  }
}
 
Example 12
Source Project: hadoop-gpu   File: Server.java    License: Apache License 2.0 5 votes vote down vote up
static Class<?> getProtocolClass(String protocolName, Configuration conf) 
throws ClassNotFoundException {
  Class<?> protocol = PROTOCOL_CACHE.get(protocolName);
  if (protocol == null) {
    protocol = conf.getClassByName(protocolName);
    PROTOCOL_CACHE.put(protocolName, protocol);
  }
  return protocol;
}
 
Example 13
@VisibleForTesting
public static String getAccountKeyFromConfiguration(String accountName,
    Configuration conf) throws KeyProviderException {
  String key = null;
  String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX
      + accountName);
  KeyProvider keyProvider = null;

  if (keyProviderClass == null) {
    // No key provider was provided so use the provided key as is.
    keyProvider = new SimpleKeyProvider();
  } else {
    // create an instance of the key provider class and verify it
    // implements KeyProvider
    Object keyProviderObject = null;
    try {
      Class<?> clazz = conf.getClassByName(keyProviderClass);
      keyProviderObject = clazz.newInstance();
    } catch (Exception e) {
      throw new KeyProviderException("Unable to load key provider class.", e);
    }
    if (!(keyProviderObject instanceof KeyProvider)) {
      throw new KeyProviderException(keyProviderClass
          + " specified in config is not a valid KeyProvider class.");
    }
    keyProvider = (KeyProvider) keyProviderObject;
  }
  key = keyProvider.getStorageAccountKey(accountName, conf);

  return key;
}
 
Example 14
Source Project: big-c   File: NetUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the socket factory corresponding to the given proxy URI. If the
 * given proxy URI corresponds to an absence of configuration parameter,
 * returns null. If the URI is malformed raises an exception.
 * 
 * @param propValue the property which is the class name of the
 *        SocketFactory to instantiate; assumed non null and non empty.
 * @return a socket factory as defined in the property value.
 */
public static SocketFactory getSocketFactoryFromProperty(
    Configuration conf, String propValue) {

  try {
    Class<?> theClass = conf.getClassByName(propValue);
    return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);

  } catch (ClassNotFoundException cnfe) {
    throw new RuntimeException("Socket Factory class not found: " + cnfe);
  }
}
 
Example 15
Source Project: big-c   File: SerializationFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
    ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serialization class not found: ", e);
  }
}
 
Example 16
Source Project: tez   File: MRInput.java    License: Apache License 2.0 5 votes vote down vote up
MRInputConfigBuilder(Configuration conf, Class<?> inputFormatParam) {
  this.conf = conf;
  if (inputFormatParam != null) {
    inputFormatProvided = true;
    this.inputFormat = inputFormatParam;
    if (org.apache.hadoop.mapred.InputFormat.class.isAssignableFrom(inputFormatParam)) {
      useNewApi = false;
    } else if (org.apache.hadoop.mapreduce.InputFormat.class.isAssignableFrom(inputFormatParam)) {
      useNewApi = true;
    } else {
      throw new TezUncheckedException("inputFormat must be assignable from either " +
          "org.apache.hadoop.mapred.InputFormat or " +
          "org.apache.hadoop.mapreduce.InputFormat" +
          " Given: " + inputFormatParam.getName());
    }
  } else {
    inputFormatProvided = false;
    useNewApi = conf.getBoolean(MRJobConfig.NEW_API_MAPPER_CONFIG, true);
    try {
      if (useNewApi) {
        this.inputFormat = conf.getClassByName(conf.get(MRJobConfig.INPUT_FORMAT_CLASS_ATTR));
        Preconditions.checkState(org.apache.hadoop.mapreduce.InputFormat.class
            .isAssignableFrom(this.inputFormat));
      } else {
        this.inputFormat = conf.getClassByName(conf.get("mapred.input.format.class"));
        Preconditions.checkState(org.apache.hadoop.mapred.InputFormat.class
            .isAssignableFrom(this.inputFormat));
      }
    } catch (ClassNotFoundException e) {
      throw new TezUncheckedException(e);
    }
    initializeInputPath();
  }
}
 
Example 17
Source Project: RDFS   File: SerializationFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
        ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serilization class not found: " +
        StringUtils.stringifyException(e));
  }
}
 
Example 18
Source Project: hadoop   File: ReliabilityTest.java    License: Apache License 2.0 4 votes vote down vote up
private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.jobsToComplete()).length == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  if(rJob.isComplete()) {
    LOG.error("The last job returned by the querying JobTracker is complete :" + 
        rJob.getJobID() + " .Exiting the test");
    System.exit(-1);
  }
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}
 
Example 19
Source Project: hadoop-gpu   File: ReliabilityTest.java    License: Apache License 2.0 4 votes vote down vote up
private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  int prevJobsNum = jc.getAllJobs().length;
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.getAllJobs()).length - prevJobsNum == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}
 
Example 20
Source Project: RDFS   File: ReliabilityTest.java    License: Apache License 2.0 4 votes vote down vote up
private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.jobsToComplete()).length == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  if(rJob.isComplete()) {
    LOG.error("The last job returned by the querying JobTracker is complete :" + 
        rJob.getJobID() + " .Exiting the test");
    System.exit(-1);
  }
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}