Java Code Examples for org.apache.hadoop.conf.Configuration#iterator()

The following examples show how to use org.apache.hadoop.conf.Configuration#iterator() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static Properties extractKafkaConfigToProperties(Configuration configuration) {
    Set<String> configNames = new HashSet<String>();
    try {
        configNames = ConsumerConfig.configNames();
    } catch (Exception e) {
        // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException which is an Error, not Exception
        String[] configNamesArray = ("metric.reporters, metadata.max.age.ms, partition.assignment.strategy, reconnect.backoff.ms," + "sasl.kerberos.ticket.renew.window.factor, max.partition.fetch.bytes, bootstrap.servers, ssl.keystore.type," + " enable.auto.commit, sasl.mechanism, interceptor.classes, exclude.internal.topics, ssl.truststore.password," + " client.id, ssl.endpoint.identification.algorithm, max.poll.records, check.crcs, request.timeout.ms, heartbeat.interval.ms," + " auto.commit.interval.ms, receive.buffer.bytes, ssl.truststore.type, ssl.truststore.location, ssl.keystore.password, fetch.min.bytes," + " fetch.max.bytes, send.buffer.bytes, max.poll.interval.ms, value.deserializer, group.id, retry.backoff.ms,"
                + " ssl.secure.random.implementation, sasl.kerberos.kinit.cmd, sasl.kerberos.service.name, sasl.kerberos.ticket.renew.jitter, ssl.trustmanager.algorithm, ssl.key.password, fetch.max.wait.ms, sasl.kerberos.min.time.before.relogin, connections.max.idle.ms, session.timeout.ms, metrics.num.samples, key.deserializer, ssl.protocol, ssl.provider, ssl.enabled.protocols, ssl.keystore.location, ssl.cipher.suites, security.protocol, ssl.keymanager.algorithm, metrics.sample.window.ms, auto.offset.reset").split(",");
        configNames.addAll(Arrays.asList(configNamesArray));
    }

    Properties result = new Properties();
    for (Iterator<Map.Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
        Map.Entry<String, String> entry = it.next();
        String key = entry.getKey();
        String value = entry.getValue();
        if (configNames.contains(key)) {
            result.put(key, value);
        }
    }
    return result;
}
 
Example 2
Source Project: kylin   File: KafkaConsumerProperties.java    License: Apache License 2.0 6 votes vote down vote up
public static Properties extractKafkaConfigToProperties(Configuration configuration) {
    Set<String> configNames = new HashSet<String>();
    try {
        configNames = ConsumerConfig.configNames();
    } catch (Exception e) {
        // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException which is an Error, not Exception
        String[] configNamesArray = ("metric.reporters, metadata.max.age.ms, partition.assignment.strategy, reconnect.backoff.ms," + "sasl.kerberos.ticket.renew.window.factor, max.partition.fetch.bytes, bootstrap.servers, ssl.keystore.type," + " enable.auto.commit, sasl.mechanism, interceptor.classes, exclude.internal.topics, ssl.truststore.password," + " client.id, ssl.endpoint.identification.algorithm, max.poll.records, check.crcs, request.timeout.ms, heartbeat.interval.ms," + " auto.commit.interval.ms, receive.buffer.bytes, ssl.truststore.type, ssl.truststore.location, ssl.keystore.password, fetch.min.bytes," + " fetch.max.bytes, send.buffer.bytes, max.poll.interval.ms, value.deserializer, group.id, retry.backoff.ms,"
                + " ssl.secure.random.implementation, sasl.kerberos.kinit.cmd, sasl.kerberos.service.name, sasl.kerberos.ticket.renew.jitter, ssl.trustmanager.algorithm, ssl.key.password, fetch.max.wait.ms, sasl.kerberos.min.time.before.relogin, connections.max.idle.ms, session.timeout.ms, metrics.num.samples, key.deserializer, ssl.protocol, ssl.provider, ssl.enabled.protocols, ssl.keystore.location, ssl.cipher.suites, security.protocol, ssl.keymanager.algorithm, metrics.sample.window.ms, auto.offset.reset").split(",");
        configNames.addAll(Arrays.asList(configNamesArray));
    }

    Properties result = new Properties();
    for (Iterator<Map.Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
        Map.Entry<String, String> entry = it.next();
        String key = entry.getKey();
        String value = entry.getValue();
        if (configNames.contains(key)) {
            result.put(key, value);
        }
    }
    return result;
}
 
Example 3
Source Project: ambari-shell   File: ConfigCommands.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets the desired configuration.
 */
@CliCommand(value = "configuration set", help = "Sets the desired configuration")
public String setConfig(@CliOption(key = "type", mandatory = true, help = "Type of the configuration") ConfigType configType,
                        @CliOption(key = "url", help = "URL of the config") String url,
                        @CliOption(key = "file", help = "File of the config") File file) throws IOException {
  Configuration configuration = new Configuration(false);
  if (file == null) {
    configuration.addResource(new URL(url));
  } else {
    configuration.addResource(new FileInputStream(file));
  }
  Map<String, String> config = new HashMap<String, String>();
  Iterator<Map.Entry<String, String>> iterator = configuration.iterator();
  while (iterator.hasNext()) {
    Map.Entry<String, String> entry = iterator.next();
    config.put(entry.getKey(), entry.getValue());
  }
  client.modifyConfiguration(configType.getName(), config);
  return "Restart is required!\n" + renderSingleMap(config, "KEY", "VALUE");
}
 
Example 4
Source Project: spork   File: TezDagBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public static String convertToHistoryText(String description, Configuration conf) throws IOException {
    // Add a version if this serialization is changed
    JSONObject jsonObject = new JSONObject();
    try {
        if (description != null && !description.isEmpty()) {
            jsonObject.put("desc", description);
    }
    if (conf != null) {
        JSONObject confJson = new JSONObject();
        Iterator<Entry<String, String>> iter = conf.iterator();
        while (iter.hasNext()) {
            Entry<String, String> entry = iter.next();
            confJson.put(entry.getKey(), entry.getValue());
        }
        jsonObject.put("config", confJson);
    }
    } catch (JSONException e) {
        throw new IOException("Error when trying to convert description/conf to JSON", e);
    }
    return jsonObject.toString();
}
 
Example 5
Source Project: spork   File: AbstractAccumuloStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf,
        Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}
 
Example 6
Source Project: Bats   File: LogicalPlanConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
private static Properties toProperties(Configuration conf)
{
  Iterator<Entry<String, String>> it = conf.iterator();
  Properties props = new Properties();
  while (it.hasNext()) {
    Entry<String, String> e = it.next();
    props.put(e.getKey(), e.getValue());
  }
  return props;
}
 
Example 7
Source Project: pxf   File: JdbcBasePlugin.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs a mapping of configuration and includes all properties that start with the specified
 * configuration prefix.  Property names in the mapping are trimmed to remove the configuration prefix.
 * This is a method from Hadoop's Configuration class ported here to make older and custom versions of Hadoop
 * work with JDBC profile.
 *
 * @param configuration configuration map
 * @param confPrefix    configuration prefix
 * @return mapping of configuration properties with prefix stripped
 */
private Map<String, String> getPropsWithPrefix(Configuration configuration, String confPrefix) {
    Map<String, String> configMap = new HashMap<>();
    Iterator<Map.Entry<String, String>> it = configuration.iterator();
    while (it.hasNext()) {
        String propertyName = it.next().getKey();
        if (propertyName.startsWith(confPrefix)) {
            // do not use value from the iterator as it might not come with variable substitution
            String value = configuration.get(propertyName);
            String keyName = propertyName.substring(confPrefix.length());
            configMap.put(keyName, value);
        }
    }
    return configMap;
}
 
Example 8
Source Project: spork   File: HConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
public HConfiguration(Configuration other) {
    if (other != null) {
        Iterator<Map.Entry<String, String>> iter = other.iterator();
        
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            
            put(entry.getKey(), entry.getValue());
        }
    }
}
 
Example 9
/**
 * Utility to work around the limitation of the copy constructor
 * {@link Configuration#Configuration(Configuration)} provided by the {@link Configuration}
 * class. See https://issues.apache.org/jira/browse/HBASE-18378.
 * The copy constructor doesn't copy all the config settings, so we need to resort to
 * iterating through all the settings and setting it on the cloned config.
 * @param toCopy  configuration to copy
 * @return
 */
private static Configuration cloneConfig(Configuration toCopy) {
    Configuration clone = new Configuration();
    Iterator<Entry<String, String>> iterator = toCopy.iterator();
    while (iterator.hasNext()) {
        Entry<String, String> entry = iterator.next();
        clone.set(entry.getKey(), entry.getValue());
    }
    return clone;
}
 
Example 10
Source Project: hraven   File: JobHistoryService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the HBase {@code Put} instances to store for the given
 * {@code Configuration} data. Each configuration property will be stored as a
 * separate key value.
 *
 * @param jobDesc the {@link JobDesc} generated for the job
 * @param jobConf the job configuration
 * @return puts for the given job configuration
 */
public static List<Put> getHbasePuts(JobDesc jobDesc, Configuration jobConf) {
  List<Put> puts = new LinkedList<Put>();

  JobKey jobKey = new JobKey(jobDesc);
  byte[] jobKeyBytes = new JobKeyConverter().toBytes(jobKey);

  // Add all columns to one put
  Put jobPut = new Put(jobKeyBytes);
  jobPut.addColumn(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES,
      Bytes.toBytes(jobDesc.getVersion()));
  jobPut.addColumn(Constants.INFO_FAM_BYTES, Constants.FRAMEWORK_COLUMN_BYTES,
      Bytes.toBytes(jobDesc.getFramework().toString()));

  // Avoid doing string to byte conversion inside loop.
  byte[] jobConfColumnPrefix =
      Bytes.toBytes(Constants.JOB_CONF_COLUMN_PREFIX + Constants.SEP);

  // Create puts for all the parameters in the job configuration
  Iterator<Entry<String, String>> jobConfIterator = jobConf.iterator();
  while (jobConfIterator.hasNext()) {
    Entry<String, String> entry = jobConfIterator.next();
    // Prefix the job conf entry column with an indicator to
    byte[] column =
        Bytes.add(jobConfColumnPrefix, Bytes.toBytes(entry.getKey()));
    jobPut.addColumn(Constants.INFO_FAM_BYTES, column,
        Bytes.toBytes(entry.getValue()));
  }

  // ensure pool/queuename is set correctly
  setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);

  puts.add(jobPut);

  return puts;
}
 
Example 11
public void testBug50574() throws Exception {
  Properties props = new Properties();
  int mcastPort = AvailablePort.getRandomAvailablePort(AvailablePort.JGROUPS);
  props.put("mcast-port", String.valueOf(mcastPort));
  Connection conn = TestUtil.getConnection(props);
  Statement st = conn.createStatement();
  
  
  st.execute(" create hdfsstore sensorStore  NameNode 'localhost'  HomeDir './sensorStore' " +
      "BatchTimeInterval 10 milliseconds WriteOnlyFileRolloverInterval 1 seconds");
  
  st.execute(" create table raw_sensor(   id bigint primary key,   timestamp bigint, house_id integer   ) " +
      "partition by column (house_id) persistent hdfsstore (sensorStore) writeonly");
  st.execute("insert into raw_sensor (   id,   timestamp , house_id ) values (1,1,1)");
  st.execute("insert into raw_sensor (   id,   timestamp , house_id ) values (11,11,11)");
  Thread.sleep(2000); 
  
  TestUtil.shutDown();
  deleteOplogs();
  props.put("mcast-port", "0");
  props.put("persist-dd", "false");
  props.put(Property.HADOOP_IS_GFXD_LONER, "true");
  props.put(Property.GFXD_HD_NAMENODEURL, "localhost");
  props.put(Property.GFXD_HD_HOMEDIR, "./sensorStore");
  Configuration conf = new Configuration();
  Iterator<Entry<String, String>> confentries = conf.iterator();
  
  while (confentries.hasNext()){
    Entry<String, String> entry = confentries.next();
    props.put(Property.HADOOP_GFXD_LONER_PROPS_PREFIX + entry.getKey(), entry.getValue());
  }
  
  conn = TestUtil.getConnection(props);
  st = conn.createStatement();
  noOplogsCreatedCheck();
  
  TestUtil.shutDown();
}
 
Example 12
Source Project: incubator-tez   File: TezUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void writeConfInPB(OutputStream dos, Configuration conf) throws IOException {
  ConfigurationProto.Builder confProtoBuilder = ConfigurationProto.newBuilder();
  Iterator<Entry<String, String>> iter = conf.iterator();
  while (iter.hasNext()) {
    Entry<String, String> entry = iter.next();
    PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
    kvp.setKey(entry.getKey());
    kvp.setValue(entry.getValue());
    confProtoBuilder.addConfKeyValues(kvp);
  }
  ConfigurationProto confProto = confProtoBuilder.build();
  confProto.writeTo(dos);
}
 
Example 13
public SerializableHadoopConfig(@NonNull Configuration configuration){
    this.configuration = configuration;
    content = new LinkedHashMap<>();
    Iterator<Map.Entry<String,String>> iter = configuration.iterator();
    while(iter.hasNext()){
        Map.Entry<String,String> next = iter.next();
        content.put(next.getKey(), next.getValue());
    }
}
 
Example 14
private Configuration filterByBlacklist(Configuration originalConf, List<String> blacklist) {
    if (blacklist != null && blacklist.size() > 0) {
        Configuration filteredConf = new Configuration(false);
        Iterator<Entry<String, String>> iterator = originalConf.iterator();
        while (iterator.hasNext()) {
            Entry<String, String> next = iterator.next();
            if (blacklist.contains(next.getKey())) {
                continue;
            }
            filteredConf.set(next.getKey(), next.getValue());
        }
        originalConf = filteredConf;
    }
    return originalConf;
}
 
Example 15
Source Project: hbase   File: TestBackupBase.java    License: Apache License 2.0 5 votes vote down vote up
private static void populateFromMasterConfig(Configuration masterConf, Configuration conf) {
  Iterator<Entry<String, String>> it = masterConf.iterator();
  while (it.hasNext()) {
    Entry<String, String> e = it.next();
    conf.set(e.getKey(), e.getValue());
  }
}
 
Example 16
protected Map<String, String> getContents(Configuration conf) {
    Map<String, String> contents = new HashMap<String, String>();
    Iterator<Entry<String, String>> iter = conf.iterator();
    while (iter.hasNext()) {
        Entry<String, String> entry = iter.next();

        contents.put(entry.getKey(), entry.getValue());
    }

    return contents;
}
 
Example 17
/**
  * Pull properties and values from filename.
  *
  * @param filename XML filename
  * @return HashMap containing <Property,Value> entries from XML file
  */
 private HashMap<String,String> extractPropertiesFromXml
     (String filename) {
   if (filename==null) {
     return null;
   }

   // Iterate through XML file for name/value pairs
   Configuration conf = new Configuration(false);
   conf.setAllowNullValueProperties(true);
   conf.addResource(filename);

   HashMap<String,String> retVal = new HashMap<String,String>();
   Iterator<Map.Entry<String,String>> kvItr = conf.iterator();
   while (kvItr.hasNext()) {
     Map.Entry<String,String> entry = kvItr.next();
     String key = entry.getKey();
     // Ignore known xml props
     if (xmlPropsToSkipCompare != null) {
       if (xmlPropsToSkipCompare.contains(key)) {
         continue;
       }
     }
     // Ignore known xml prefixes
     boolean skipPrefix = false;
     if (xmlPrefixToSkipCompare != null) {
       for (String xmlPrefix : xmlPrefixToSkipCompare) {
  if (key.startsWith(xmlPrefix)) {
    skipPrefix = true;
           break;
  }
}
     }
     if (skipPrefix) {
       continue;
     }
     if (conf.onlyKeyExists(key)) {
       retVal.put(key,null);
     } else {
       String value = conf.get(key);
       if (value!=null) {
         retVal.put(key,entry.getValue());
       }
     }
     kvItr.remove();
   }
   return retVal;
 }
 
Example 18
public void setParameters(Configuration conf) {
  for( Iterator<Entry<String, String>> iter = conf.iterator(); iter.hasNext();) {
    Map.Entry<String, String> entry = iter.next();
    this.hiveConf.set(entry.getKey(), entry.getValue());
  }
}
 
Example 19
public static Configuration combineConfigurations(Configuration primary, Map<String, Configuration> failovers ) {

    Configuration resultingConfig = new Configuration();
    resultingConfig.clear();

    boolean isFirst = true;
    StringBuilder failOverClusterNames = new StringBuilder();


    Iterator<Entry<String, String>> primaryIt =  primary.iterator();

    while(primaryIt.hasNext()) {
      Entry<String, String> primaryKeyValue = primaryIt.next();
      resultingConfig.set(primaryKeyValue.getKey().replace('_', '.'), primaryKeyValue.getValue());
    }


    for (Entry<String, Configuration> failover: failovers.entrySet()) {
      if (isFirst) {
        isFirst = false;
      } else {
        failOverClusterNames.append(",");
      }
      failOverClusterNames.append(failover.getKey());

      Configuration failureConfig = failover.getValue();

      Iterator<Entry<String, String>> it = failureConfig.iterator();
      while (it.hasNext()) {
        Entry<String, String> keyValue = it.next();

        LOG.info(" -- Looking at : " + keyValue.getKey() + "=" + keyValue.getValue());

        String configKey = keyValue.getKey().replace('_', '.');

        if (configKey.startsWith("hbase.")) {
          resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTER_CONFIG + "." + failover.getKey() + "." + configKey , keyValue.getValue());
          LOG.info(" - Porting config: " + configKey + "=" + keyValue.getValue());
        }
      }
    }

    resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG, failOverClusterNames.toString());

    return resultingConfig;
  }
 
Example 20
/**
 * Put all configuration properties in a given {@link Configuration} object into a given
 * {@link Properties} object.
 *
 * @param configuration the given {@link Configuration} object
 * @param properties the given {@link Properties} object
 */
public static void putConfigurationIntoProperties(Configuration configuration, Properties properties) {
  for (Iterator<Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
    Entry<String, String> entry = it.next();
    properties.put(entry.getKey(), entry.getValue());
  }
}