Java Code Examples for org.apache.hadoop.conf.Configuration#getClass()

The following examples show how to use org.apache.hadoop.conf.Configuration#getClass() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: kite   File: DatasetKeyOutputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <E> Class<E> getType(JobContext jobContext) {
  Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext);
  Class<E> type;
  try {
    type = (Class<E>)conf.getClass(KITE_TYPE, GenericData.Record.class);
  } catch (RuntimeException e) {
    if (e.getCause() instanceof ClassNotFoundException) {
      throw new TypeNotFoundException(String.format(
          "The Java class %s for the entity type could not be found",
          conf.get(KITE_TYPE)),
          e.getCause());
    } else {
      throw e;
    }
  }
  return type;
}
 
Example 2
Source Project: big-c   File: Chain.java    License: Apache License 2.0 6 votes vote down vote up
protected static void checkReducerAlreadySet(boolean isMap,
    Configuration jobConf, String prefix, boolean shouldSet) {
  if (!isMap) {
    if (shouldSet) {
      if (jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, null) == null) {
        throw new IllegalStateException(
            "A Mapper can be added to the chain only after the Reducer has "
                + "been set");
      }
    } else {
      if (jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, null) != null) {
        throw new IllegalStateException("Reducer has been already set");
      }
    }
  }
}
 
Example 3
protected ApplicationHistoryStore createApplicationHistoryStore(
    Configuration conf) {
  try {
    Class<? extends ApplicationHistoryStore> storeClass =
        conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
            NullApplicationHistoryStore.class,
            ApplicationHistoryStore.class);
    return storeClass.newInstance();
  } catch (Exception e) {
    String msg =
        "Could not instantiate ApplicationHistoryWriter: "
            + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
                NullApplicationHistoryStore.class.getName());
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
}
 
Example 4
Source Project: hadoop   File: FSEditLog.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieve the implementation class for a Journal scheme.
 * @param conf The configuration to retrieve the information from
 * @param uriScheme The uri scheme to look up.
 * @return the class of the journal implementation
 * @throws IllegalArgumentException if no class is configured for uri
 */
static Class<? extends JournalManager> getJournalClass(Configuration conf,
                             String uriScheme) {
  String key
    = DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX + "." + uriScheme;
  Class <? extends JournalManager> clazz = null;
  try {
    clazz = conf.getClass(key, null, JournalManager.class);
  } catch (RuntimeException re) {
    throw new IllegalArgumentException(
        "Invalid class specified for " + uriScheme, re);
  }
    
  if (clazz == null) {
    LOG.warn("No class configured for " +uriScheme
             + ", " + key + " is empty");
    throw new IllegalArgumentException(
        "No class configured for " + uriScheme);
  }
  return clazz;
}
 
Example 5
Source Project: hadoop   File: SSLFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an SSLFactory.
 *
 * @param mode SSLFactory mode, client or server.
 * @param conf Hadoop configuration from where the SSLFactory configuration
 * will be read.
 */
public SSLFactory(Mode mode, Configuration conf) {
  this.conf = conf;
  if (mode == null) {
    throw new IllegalArgumentException("mode cannot be NULL");
  }
  this.mode = mode;
  requireClientCert = conf.getBoolean(SSL_REQUIRE_CLIENT_CERT_KEY,
                                      DEFAULT_SSL_REQUIRE_CLIENT_CERT);
  Configuration sslConf = readSSLConfiguration(mode);

  Class<? extends KeyStoresFactory> klass
    = conf.getClass(KEYSTORES_FACTORY_CLASS_KEY,
                    FileBasedKeyStoresFactory.class, KeyStoresFactory.class);
  keystoresFactory = ReflectionUtils.newInstance(klass, sslConf);

  enabledProtocols = conf.getStrings(SSL_ENABLED_PROTOCOLS,
      DEFAULT_SSL_ENABLED_PROTOCOLS);
}
 
Example 6
protected void setupContextWriter(Configuration conf) throws IOException {
    Class<ContextWriter<K2,V2>> contextWriterClass = null;
    if (Mutation.class.equals(conf.getClass(MAPRED_OUTPUT_VALUE_CLASS, null))) {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, LiveContextWriter.class, ContextWriter.class);
    } else {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, BulkContextWriter.class, ContextWriter.class);
    }
    try {
        setContextWriter(contextWriterClass.newInstance());
        contextWriter.setup(conf, conf.getBoolean(CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, false));
    } catch (Exception e) {
        throw new IOException("Failed to initialized " + contextWriterClass + " from property " + CONTEXT_WRITER_CLASS, e);
    }
}
 
Example 7
protected void setupContextWriter(Configuration conf) throws IOException {
    Class<ContextWriter<K2,V2>> contextWriterClass = null;
    if (Mutation.class.equals(conf.getClass(MAPRED_OUTPUT_VALUE_CLASS, null))) {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, LiveContextWriter.class, ContextWriter.class);
    } else {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, BulkContextWriter.class, ContextWriter.class);
    }
    try {
        setContextWriter(contextWriterClass.newInstance());
        contextWriter.setup(conf, conf.getBoolean(CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, false));
    } catch (Exception e) {
        throw new IOException("Failed to initialized " + contextWriterClass + " from property " + CONTEXT_WRITER_CLASS, e);
    }
}
 
Example 8
Source Project: hadoop   File: FsDatasetSpi.java    License: Apache License 2.0 5 votes vote down vote up
/** @return the configured factory. */
public static Factory<?> getFactory(Configuration conf) {
  @SuppressWarnings("rawtypes")
  final Class<? extends Factory> clazz = conf.getClass(
      DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY,
      FsDatasetFactory.class,
      Factory.class);
  return ReflectionUtils.newInstance(clazz, conf);
}
 
Example 9
Source Project: hbase   File: BackupRestoreFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets backup merge job
 * @param conf configuration
 * @return backup merge job instance
 */
public static BackupMergeJob getBackupMergeJob(Configuration conf) {
  Class<? extends BackupMergeJob> cls =
      conf.getClass(HBASE_BACKUP_MERGE_IMPL_CLASS, MapReduceBackupMergeJob.class,
        BackupMergeJob.class);
  BackupMergeJob service = ReflectionUtils.newInstance(cls, conf);
  service.setConf(conf);
  return service;
}
 
Example 10
Source Project: hbase   File: BackupRestoreFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets backup copy job
 * @param conf configuration
 * @return backup copy job instance
 */
public static BackupCopyJob getBackupCopyJob(Configuration conf) {
  Class<? extends BackupCopyJob> cls =
      conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyJob.class,
        BackupCopyJob.class);
  BackupCopyJob service = ReflectionUtils.newInstance(cls, conf);
  service.setConf(conf);
  return service;
}
 
Example 11
Source Project: hbase   File: TestFromClientSide4.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * test of that unmanaged HConnections are able to reconnect
 * properly (see HBASE-5058)
 */
@Test public void testUnmanagedHConnectionReconnect() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  Class registryImpl = conf
    .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
  // This test does not make sense for MasterRegistry since it stops the only master in the
  // cluster and starts a new master without populating the underlying config for the connection.
  Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
  final TableName tableName = name.getTableName();
  TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
  try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
    try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
      assertTrue(admin.tableExists(tableName));
      assertTrue(t.get(new Get(ROW)).isEmpty());
    }

    // stop the master
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    cluster.stopMaster(0, false);
    cluster.waitOnMaster(0);

    // start up a new master
    cluster.startMaster();
    assertTrue(cluster.waitForActiveAndReadyMaster());

    // test that the same unmanaged connection works with a new
    // Admin and can connect to the new master;
    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
    try (Admin admin = conn.getAdmin()) {
      assertTrue(admin.tableExists(tableName));
      assertEquals(
        admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
        SLAVES + (tablesOnMaster ? 1 : 0));
    }
  }
}
 
Example 12
public KeyValueReader(Configuration conf) {
    super(conf);
    keyClass = conf.getClass(INPUT_KEY_CLASS, Text.class, 
            WritableComparable.class);
    valueClass = conf.getClass(INPUT_VALUE_CLASS, Text.class, 
            Writable.class);    
}
 
Example 13
Source Project: hbase   File: WALFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param conf must not be null, will keep a reference to read params in later reader/writer
 *          instances.
 * @param factoryId a unique identifier for this factory. used i.e. by filesystem implementations
 *          to make a directory
 * @param enableSyncReplicationWALProvider whether wrap the wal provider to a
 *          {@link SyncReplicationWALProvider}
 */
public WALFactory(Configuration conf, String factoryId, boolean enableSyncReplicationWALProvider)
    throws IOException {
  // until we've moved reader/writer construction down into providers, this initialization must
  // happen prior to provider initialization, in case they need to instantiate a reader/writer.
  timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
  /* TODO Both of these are probably specific to the fs wal provider */
  logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
    AbstractFSWALProvider.Reader.class);
  this.conf = conf;
  this.factoryId = factoryId;
  // end required early initialization
  if (conf.getBoolean(WAL_ENABLED, true)) {
    WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
    if (enableSyncReplicationWALProvider) {
      provider = new SyncReplicationWALProvider(provider);
    }
    provider.init(this, conf, null);
    provider.addWALActionsListener(new MetricsWAL());
    this.provider = provider;
  } else {
    // special handling of existing configuration behavior.
    LOG.warn("Running with WAL disabled.");
    provider = new DisabledWALProvider();
    provider.init(this, conf, factoryId);
  }
}
 
Example 14
Source Project: incubator-tez   File: ConfigUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static <V> RawComparator<V> getInputKeySecondaryGroupingComparator(
    Configuration conf) {
  Class<? extends RawComparator> theClass = conf
      .getClass(
          TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS,
          null, RawComparator.class);
  if (theClass == null) {
    return getIntermediateInputKeyComparator(conf);
  }

  return ReflectionUtils.newInstance(theClass, conf);
}
 
Example 15
Source Project: hadoop   File: BlockPlacementPolicy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get an instance of the configured Block Placement Policy based on the
 * the configuration property
 * {@link  DFSConfigKeys#DFS_BLOCK_REPLICATOR_CLASSNAME_KEY}.
 * 
 * @param conf the configuration to be used
 * @param stats an object that is used to retrieve the load on the cluster
 * @param clusterMap the network topology of the cluster
 * @return an instance of BlockPlacementPolicy
 */
public static BlockPlacementPolicy getInstance(Configuration conf, 
                                               FSClusterStats stats,
                                               NetworkTopology clusterMap,
                                               Host2NodesMap host2datanodeMap) {
  final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
      BlockPlacementPolicy.class);
  final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
      replicatorClass, conf);
  replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
  return replicator;
}
 
Example 16
Source Project: RDFS   File: JobTracker.java    License: Apache License 2.0 4 votes vote down vote up
public static Class<? extends JobTrackerInstrumentation> getInstrumentationClass(Configuration conf) {
  return conf.getClass("mapred.jobtracker.instrumentation",
      JobTrackerMetricsInst.class, JobTrackerInstrumentation.class);
}
 
Example 17
Source Project: zerowing   File: ConfigUtil.java    License: MIT License 4 votes vote down vote up
public static Class<? extends Translator> getTranslatorClass(Configuration conf) {
  return conf.getClass(TRANSLATOR_CLASS, BasicTranslator.class, Translator.class);
}
 
Example 18
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
 
Example 19
Source Project: incubator-tez   File: ConfigUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static <K> Class<K> getIntermediateInputKeyClass(Configuration conf) {
  Class<K> retv = (Class<K>) conf.getClass(
      TezJobConfig.TEZ_RUNTIME_KEY_CLASS, null,
      Object.class);
  return retv;
}
 
Example 20
Source Project: big-c   File: SaslPropertiesResolver.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Returns an instance of SaslPropertiesResolver.
 * Looks up the configuration to see if there is custom class specified.
 * Constructs the instance by passing the configuration directly to the
 * constructor to achieve thread safety using final fields.
 * @param conf
 * @return SaslPropertiesResolver
 */
public static SaslPropertiesResolver getInstance(Configuration conf) {
  Class<? extends SaslPropertiesResolver> clazz =
    conf.getClass(
        CommonConfigurationKeysPublic.HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
        SaslPropertiesResolver.class, SaslPropertiesResolver.class);
  return ReflectionUtils.newInstance(clazz, conf);
}