Java Code Examples for org.apache.hadoop.conf.Configuration#getClass()

The following examples show how to use org.apache.hadoop.conf.Configuration#getClass() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DatasetKeyOutputFormat.java    From kite with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <E> Class<E> getType(JobContext jobContext) {
  Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext);
  Class<E> type;
  try {
    type = (Class<E>)conf.getClass(KITE_TYPE, GenericData.Record.class);
  } catch (RuntimeException e) {
    if (e.getCause() instanceof ClassNotFoundException) {
      throw new TypeNotFoundException(String.format(
          "The Java class %s for the entity type could not be found",
          conf.get(KITE_TYPE)),
          e.getCause());
    } else {
      throw e;
    }
  }
  return type;
}
 
Example 2
Source File: Chain.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected static void checkReducerAlreadySet(boolean isMap,
    Configuration jobConf, String prefix, boolean shouldSet) {
  if (!isMap) {
    if (shouldSet) {
      if (jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, null) == null) {
        throw new IllegalStateException(
            "A Mapper can be added to the chain only after the Reducer has "
                + "been set");
      }
    } else {
      if (jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, null) != null) {
        throw new IllegalStateException("Reducer has been already set");
      }
    }
  }
}
 
Example 3
Source File: RMApplicationHistoryWriter.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected ApplicationHistoryStore createApplicationHistoryStore(
    Configuration conf) {
  try {
    Class<? extends ApplicationHistoryStore> storeClass =
        conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
            NullApplicationHistoryStore.class,
            ApplicationHistoryStore.class);
    return storeClass.newInstance();
  } catch (Exception e) {
    String msg =
        "Could not instantiate ApplicationHistoryWriter: "
            + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
                NullApplicationHistoryStore.class.getName());
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
}
 
Example 4
Source File: SSLFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an SSLFactory.
 *
 * @param mode SSLFactory mode, client or server.
 * @param conf Hadoop configuration from where the SSLFactory configuration
 * will be read.
 */
public SSLFactory(Mode mode, Configuration conf) {
  this.conf = conf;
  if (mode == null) {
    throw new IllegalArgumentException("mode cannot be NULL");
  }
  this.mode = mode;
  requireClientCert = conf.getBoolean(SSL_REQUIRE_CLIENT_CERT_KEY,
                                      DEFAULT_SSL_REQUIRE_CLIENT_CERT);
  Configuration sslConf = readSSLConfiguration(mode);

  Class<? extends KeyStoresFactory> klass
    = conf.getClass(KEYSTORES_FACTORY_CLASS_KEY,
                    FileBasedKeyStoresFactory.class, KeyStoresFactory.class);
  keystoresFactory = ReflectionUtils.newInstance(klass, sslConf);

  enabledProtocols = conf.getStrings(SSL_ENABLED_PROTOCOLS,
      DEFAULT_SSL_ENABLED_PROTOCOLS);
}
 
Example 5
Source File: FSEditLog.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Retrieve the implementation class for a Journal scheme.
 * @param conf The configuration to retrieve the information from
 * @param uriScheme The uri scheme to look up.
 * @return the class of the journal implementation
 * @throws IllegalArgumentException if no class is configured for uri
 */
static Class<? extends JournalManager> getJournalClass(Configuration conf,
                             String uriScheme) {
  String key
    = DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX + "." + uriScheme;
  Class <? extends JournalManager> clazz = null;
  try {
    clazz = conf.getClass(key, null, JournalManager.class);
  } catch (RuntimeException re) {
    throw new IllegalArgumentException(
        "Invalid class specified for " + uriScheme, re);
  }
    
  if (clazz == null) {
    LOG.warn("No class configured for " +uriScheme
             + ", " + key + " is empty");
    throw new IllegalArgumentException(
        "No class configured for " + uriScheme);
  }
  return clazz;
}
 
Example 6
Source File: BlockPlacementPolicy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Get an instance of the configured Block Placement Policy based on the
 * the configuration property
 * {@link  DFSConfigKeys#DFS_BLOCK_REPLICATOR_CLASSNAME_KEY}.
 * 
 * @param conf the configuration to be used
 * @param stats an object that is used to retrieve the load on the cluster
 * @param clusterMap the network topology of the cluster
 * @return an instance of BlockPlacementPolicy
 */
public static BlockPlacementPolicy getInstance(Configuration conf, 
                                               FSClusterStats stats,
                                               NetworkTopology clusterMap,
                                               Host2NodesMap host2datanodeMap) {
  final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
      BlockPlacementPolicy.class);
  final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
      replicatorClass, conf);
  replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
  return replicator;
}
 
Example 7
Source File: ConfigUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public static <V> RawComparator<V> getInputKeySecondaryGroupingComparator(
    Configuration conf) {
  Class<? extends RawComparator> theClass = conf
      .getClass(
          TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS,
          null, RawComparator.class);
  if (theClass == null) {
    return getIntermediateInputKeyComparator(conf);
  }

  return ReflectionUtils.newInstance(theClass, conf);
}
 
Example 8
Source File: WALFactory.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @param conf must not be null, will keep a reference to read params in later reader/writer
 *          instances.
 * @param factoryId a unique identifier for this factory. used i.e. by filesystem implementations
 *          to make a directory
 * @param enableSyncReplicationWALProvider whether wrap the wal provider to a
 *          {@link SyncReplicationWALProvider}
 */
public WALFactory(Configuration conf, String factoryId, boolean enableSyncReplicationWALProvider)
    throws IOException {
  // until we've moved reader/writer construction down into providers, this initialization must
  // happen prior to provider initialization, in case they need to instantiate a reader/writer.
  timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
  /* TODO Both of these are probably specific to the fs wal provider */
  logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
    AbstractFSWALProvider.Reader.class);
  this.conf = conf;
  this.factoryId = factoryId;
  // end required early initialization
  if (conf.getBoolean(WAL_ENABLED, true)) {
    WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
    if (enableSyncReplicationWALProvider) {
      provider = new SyncReplicationWALProvider(provider);
    }
    provider.init(this, conf, null);
    provider.addWALActionsListener(new MetricsWAL());
    this.provider = provider;
  } else {
    // special handling of existing configuration behavior.
    LOG.warn("Running with WAL disabled.");
    provider = new DisabledWALProvider();
    provider.init(this, conf, factoryId);
  }
}
 
Example 9
Source File: KeyValueReader.java    From marklogic-contentpump with Apache License 2.0 5 votes vote down vote up
public KeyValueReader(Configuration conf) {
    super(conf);
    keyClass = conf.getClass(INPUT_KEY_CLASS, Text.class, 
            WritableComparable.class);
    valueClass = conf.getClass(INPUT_VALUE_CLASS, Text.class, 
            Writable.class);    
}
 
Example 10
Source File: TestFromClientSide4.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * test of that unmanaged HConnections are able to reconnect
 * properly (see HBASE-5058)
 */
@Test public void testUnmanagedHConnectionReconnect() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  Class registryImpl = conf
    .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
  // This test does not make sense for MasterRegistry since it stops the only master in the
  // cluster and starts a new master without populating the underlying config for the connection.
  Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
  final TableName tableName = name.getTableName();
  TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
  try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
    try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
      assertTrue(admin.tableExists(tableName));
      assertTrue(t.get(new Get(ROW)).isEmpty());
    }

    // stop the master
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    cluster.stopMaster(0, false);
    cluster.waitOnMaster(0);

    // start up a new master
    cluster.startMaster();
    assertTrue(cluster.waitForActiveAndReadyMaster());

    // test that the same unmanaged connection works with a new
    // Admin and can connect to the new master;
    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
    try (Admin admin = conn.getAdmin()) {
      assertTrue(admin.tableExists(tableName));
      assertEquals(
        admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
        SLAVES + (tablesOnMaster ? 1 : 0));
    }
  }
}
 
Example 11
Source File: BackupRestoreFactory.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Gets backup copy job
 * @param conf configuration
 * @return backup copy job instance
 */
public static BackupCopyJob getBackupCopyJob(Configuration conf) {
  Class<? extends BackupCopyJob> cls =
      conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyJob.class,
        BackupCopyJob.class);
  BackupCopyJob service = ReflectionUtils.newInstance(cls, conf);
  service.setConf(conf);
  return service;
}
 
Example 12
Source File: BackupRestoreFactory.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Gets backup merge job
 * @param conf configuration
 * @return backup merge job instance
 */
public static BackupMergeJob getBackupMergeJob(Configuration conf) {
  Class<? extends BackupMergeJob> cls =
      conf.getClass(HBASE_BACKUP_MERGE_IMPL_CLASS, MapReduceBackupMergeJob.class,
        BackupMergeJob.class);
  BackupMergeJob service = ReflectionUtils.newInstance(cls, conf);
  service.setConf(conf);
  return service;
}
 
Example 13
Source File: FsDatasetSpi.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** @return the configured factory. */
public static Factory<?> getFactory(Configuration conf) {
  @SuppressWarnings("rawtypes")
  final Class<? extends Factory> clazz = conf.getClass(
      DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY,
      FsDatasetFactory.class,
      Factory.class);
  return ReflectionUtils.newInstance(clazz, conf);
}
 
Example 14
Source File: BulkIngestKeyDedupeCombiner.java    From datawave with Apache License 2.0 5 votes vote down vote up
protected void setupContextWriter(Configuration conf) throws IOException {
    Class<ContextWriter<K2,V2>> contextWriterClass = null;
    if (Mutation.class.equals(conf.getClass(MAPRED_OUTPUT_VALUE_CLASS, null))) {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, LiveContextWriter.class, ContextWriter.class);
    } else {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, BulkContextWriter.class, ContextWriter.class);
    }
    try {
        setContextWriter(contextWriterClass.newInstance());
        contextWriter.setup(conf, conf.getBoolean(CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, false));
    } catch (Exception e) {
        throw new IOException("Failed to initialized " + contextWriterClass + " from property " + CONTEXT_WRITER_CLASS, e);
    }
}
 
Example 15
Source File: BulkIngestKeyAggregatingReducer.java    From datawave with Apache License 2.0 5 votes vote down vote up
protected void setupContextWriter(Configuration conf) throws IOException {
    Class<ContextWriter<K2,V2>> contextWriterClass = null;
    if (Mutation.class.equals(conf.getClass(MAPRED_OUTPUT_VALUE_CLASS, null))) {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, LiveContextWriter.class, ContextWriter.class);
    } else {
        contextWriterClass = (Class<ContextWriter<K2,V2>>) conf.getClass(CONTEXT_WRITER_CLASS, BulkContextWriter.class, ContextWriter.class);
    }
    try {
        setContextWriter(contextWriterClass.newInstance());
        contextWriter.setup(conf, conf.getBoolean(CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, false));
    } catch (Exception e) {
        throw new IOException("Failed to initialized " + contextWriterClass + " from property " + CONTEXT_WRITER_CLASS, e);
    }
}
 
Example 16
Source File: ConfigUtil.java    From zerowing with MIT License 4 votes vote down vote up
public static Class<? extends Translator> getTranslatorClass(Configuration conf) {
  return conf.getClass(TRANSLATOR_CLASS, BasicTranslator.class, Translator.class);
}
 
Example 17
Source File: PhoenixConfigurationUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
 
Example 18
Source File: ConfigUtils.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public static <K> Class<K> getIntermediateInputKeyClass(Configuration conf) {
  Class<K> retv = (Class<K>) conf.getClass(
      TezJobConfig.TEZ_RUNTIME_KEY_CLASS, null,
      Object.class);
  return retv;
}
 
Example 19
Source File: JobTracker.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public static Class<? extends JobTrackerInstrumentation> getInstrumentationClass(Configuration conf) {
  return conf.getClass("mapred.jobtracker.instrumentation",
      JobTrackerMetricsInst.class, JobTrackerInstrumentation.class);
}
 
Example 20
Source File: SaslPropertiesResolver.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Returns an instance of SaslPropertiesResolver.
 * Looks up the configuration to see if there is custom class specified.
 * Constructs the instance by passing the configuration directly to the
 * constructor to achieve thread safety using final fields.
 * @param conf
 * @return SaslPropertiesResolver
 */
public static SaslPropertiesResolver getInstance(Configuration conf) {
  Class<? extends SaslPropertiesResolver> clazz =
    conf.getClass(
        CommonConfigurationKeysPublic.HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
        SaslPropertiesResolver.class, SaslPropertiesResolver.class);
  return ReflectionUtils.newInstance(clazz, conf);
}