Java Code Examples for org.apache.hadoop.conf.Configuration#clear()

The following examples show how to use org.apache.hadoop.conf.Configuration#clear() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}
 
Example 2
Source Project: datacollector   File: TestHdfsTarget.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyConfDirectory() throws Exception {
  // Create custom core-site.xml
  Configuration configuration = new Configuration();
  configuration.clear();
  configuration.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "file:///");
  FileOutputStream configOut = FileUtils.openOutputStream(new File(getTestDir() + "/conf-dir/core-site.xml"));
  configuration.writeXml(configOut);
  configOut.close();

  HdfsTarget hdfsTarget = HdfsTargetUtil.newBuilder()
    .hdfsUri("")
    .hdfsConfDir(getTestDir() + "/conf-dir/")
    .build();

  TargetRunner runner = new TargetRunner.Builder(HdfsDTarget.class, hdfsTarget)
    .setOnRecordError(OnRecordError.STOP_PIPELINE)
    .build();

  runner.runInit();

  // The configuration object should have the FS config from core-site.xml
  Assert.assertEquals("file:///", hdfsTarget.getHdfsConfiguration().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));

  runner.runDestroy();
}
 
Example 3
Source Project: kylin   File: MapReduceExecutableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}
 
Example 4
Source Project: spork   File: AbstractAccumuloStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf,
        Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}
 
Example 5
Source Project: datawave   File: MixedGeoAndGeoWaveTest.java    License: Apache License 2.0 5 votes vote down vote up
public static void setupConfiguration(Configuration conf) {
    conf.clear();
    
    conf.set(DATA_TYPE_NAME + BaseIngestHelper.INDEX_FIELDS, GEO_FIELD + "," + POINT_FIELD + "," + POLY_POINT_FIELD);
    conf.set(DATA_TYPE_NAME + "." + GEO_FIELD + BaseIngestHelper.FIELD_TYPE, GeoType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POLY_POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    
    conf.set(DATA_TYPE_NAME + DataTypeHelper.Properties.INGEST_POLICY_ENFORCER_CLASS, IngestPolicyEnforcer.NoOpIngestPolicyEnforcer.class.getName());
    conf.set(DataTypeHelper.Properties.DATA_NAME, DATA_TYPE_NAME);
    conf.set(TypeRegistry.INGEST_DATA_TYPES, DATA_TYPE_NAME);
    conf.set(DATA_TYPE_NAME + TypeRegistry.INGEST_HELPER, INGEST_HELPER_CLASS);
    
    conf.set(ShardedDataTypeHandler.METADATA_TABLE_NAME, TableName.METADATA);
    conf.set(ShardedDataTypeHandler.NUM_SHARDS, Integer.toString(NUM_SHARDS));
    conf.set(ShardedDataTypeHandler.SHARDED_TNAMES, TableName.SHARD + "," + TableName.ERROR_SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_TNAME, TableName.SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_LPRIORITY, "30");
    conf.set(TableName.SHARD + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_TNAME, TableName.SHARD_INDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_INDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_TNAME, TableName.SHARD_RINDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_RINDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_ENABLED, "false");
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_CONFIG, "");
    conf.set("partitioner.category.shardedTables", BalancedShardPartitioner.class.getName());
    conf.set("partitioner.category.member." + TableName.SHARD, "shardedTables");
}
 
Example 6
Source Project: ambari-metrics   File: DownSamplerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Ignore
@Test
public void testPrepareTopNDownSamplingStatement() throws Exception {
  Configuration configuration = new Configuration();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "3");

  Map<String, String> conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);

  TopNDownSampler topNDownSampler = TopNDownSampler.fromConfig(conf);
  List<String> stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD_UUID");
  Assert.assertEquals(stmts.size(),2);
  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  configuration.clear();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "4");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.function", "sum");
  conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
  topNDownSampler = TopNDownSampler.fromConfig(conf);
  stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_AGGREGATE_MINUTE_UUID");
  Assert.assertEquals(stmts.size(),1);

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "SUM(METRIC_SUM), 1, SUM(METRIC_SUM), SUM(METRIC_SUM) FROM METRIC_AGGREGATE_MINUTE_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4"));
}
 
Example 7
Source Project: mr4c   File: S3CredentialsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testConfigurationUpdate() throws Exception {
	S3Credentials cred1 = buildS3Credentials1();
	Configuration conf = new Configuration();
	conf.clear();
	cred1.applyTo(conf);
	S3Credentials cred2 = S3Credentials.extractFrom(conf);
	assertEquals(cred1, cred2);
}
 
Example 8
Source Project: hbase   File: TestHQuorumPeer.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testShouldAssignDefaultZookeeperClientPort() {
  Configuration config = HBaseConfiguration.create();
  config.clear();
  Properties p = ZKConfig.makeZKProps(config);
  assertNotNull(p);
  assertEquals(2181, p.get("clientPort"));
}
 
Example 9
Source Project: incubator-tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 10
Source Project: incubator-tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  byte[] bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 11
Source Project: tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=2000)
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 12
Source Project: tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=20000)
public void testByteStringToAndFromLargeConf() throws IOException {
  Configuration conf = getConf();
  int largeSizeMinimum = 64 * 1024 * 1024;
  final String alphaString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
  int largeSize = (largeSizeMinimum + alphaString.length() - 1) / alphaString.length();

  largeSize *= alphaString.length();
  assertTrue(largeSize >= alphaString.length());
  StringBuilder sb = new StringBuilder(largeSize);

  while (sb.length() < largeSize) {
    sb.append(alphaString);
  }

  String largeValue = sb.toString();
  Assert.assertEquals(largeSize, largeValue.length());
  conf.set("testLargeValue", largeValue);
  Assert.assertEquals(conf.size(), 7);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 7);
  checkConf(conf);
  Assert.assertEquals(conf.get("testLargeValue"), largeValue);
}
 
Example 13
Source Project: tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=2000)
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  UserPayload bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 14
public static Configuration combineConfigurations(Configuration primary, Map<String, Configuration> failovers ) {

    Configuration resultingConfig = new Configuration();
    resultingConfig.clear();

    boolean isFirst = true;
    StringBuilder failOverClusterNames = new StringBuilder();


    Iterator<Entry<String, String>> primaryIt =  primary.iterator();

    while(primaryIt.hasNext()) {
      Entry<String, String> primaryKeyValue = primaryIt.next();
      resultingConfig.set(primaryKeyValue.getKey().replace('_', '.'), primaryKeyValue.getValue());
    }


    for (Entry<String, Configuration> failover: failovers.entrySet()) {
      if (isFirst) {
        isFirst = false;
      } else {
        failOverClusterNames.append(",");
      }
      failOverClusterNames.append(failover.getKey());

      Configuration failureConfig = failover.getValue();

      Iterator<Entry<String, String>> it = failureConfig.iterator();
      while (it.hasNext()) {
        Entry<String, String> keyValue = it.next();

        LOG.info(" -- Looking at : " + keyValue.getKey() + "=" + keyValue.getValue());

        String configKey = keyValue.getKey().replace('_', '.');

        if (configKey.startsWith("hbase.")) {
          resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTER_CONFIG + "." + failover.getKey() + "." + configKey , keyValue.getValue());
          LOG.info(" - Porting config: " + configKey + "=" + keyValue.getValue());
        }
      }
    }

    resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG, failOverClusterNames.toString());

    return resultingConfig;
  }