Java Code Examples for org.apache.hadoop.conf.Configuration#clear()

The following examples show how to use org.apache.hadoop.conf.Configuration#clear() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapReduceExecutableTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}
 
Example 2
Source File: TestHdfsTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyConfDirectory() throws Exception {
  // Create custom core-site.xml
  Configuration configuration = new Configuration();
  configuration.clear();
  configuration.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "file:///");
  FileOutputStream configOut = FileUtils.openOutputStream(new File(getTestDir() + "/conf-dir/core-site.xml"));
  configuration.writeXml(configOut);
  configOut.close();

  HdfsTarget hdfsTarget = HdfsTargetUtil.newBuilder()
    .hdfsUri("")
    .hdfsConfDir(getTestDir() + "/conf-dir/")
    .build();

  TargetRunner runner = new TargetRunner.Builder(HdfsDTarget.class, hdfsTarget)
    .setOnRecordError(OnRecordError.STOP_PIPELINE)
    .build();

  runner.runInit();

  // The configuration object should have the FS config from core-site.xml
  Assert.assertEquals("file:///", hdfsTarget.getHdfsConfiguration().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));

  runner.runDestroy();
}
 
Example 3
Source File: MapReduceExecutableTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}
 
Example 4
Source File: AbstractAccumuloStorage.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf,
        Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}
 
Example 5
Source File: MixedGeoAndGeoWaveTest.java    From datawave with Apache License 2.0 5 votes vote down vote up
public static void setupConfiguration(Configuration conf) {
    conf.clear();
    
    conf.set(DATA_TYPE_NAME + BaseIngestHelper.INDEX_FIELDS, GEO_FIELD + "," + POINT_FIELD + "," + POLY_POINT_FIELD);
    conf.set(DATA_TYPE_NAME + "." + GEO_FIELD + BaseIngestHelper.FIELD_TYPE, GeoType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POLY_POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    
    conf.set(DATA_TYPE_NAME + DataTypeHelper.Properties.INGEST_POLICY_ENFORCER_CLASS, IngestPolicyEnforcer.NoOpIngestPolicyEnforcer.class.getName());
    conf.set(DataTypeHelper.Properties.DATA_NAME, DATA_TYPE_NAME);
    conf.set(TypeRegistry.INGEST_DATA_TYPES, DATA_TYPE_NAME);
    conf.set(DATA_TYPE_NAME + TypeRegistry.INGEST_HELPER, INGEST_HELPER_CLASS);
    
    conf.set(ShardedDataTypeHandler.METADATA_TABLE_NAME, TableName.METADATA);
    conf.set(ShardedDataTypeHandler.NUM_SHARDS, Integer.toString(NUM_SHARDS));
    conf.set(ShardedDataTypeHandler.SHARDED_TNAMES, TableName.SHARD + "," + TableName.ERROR_SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_TNAME, TableName.SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_LPRIORITY, "30");
    conf.set(TableName.SHARD + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_TNAME, TableName.SHARD_INDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_INDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_TNAME, TableName.SHARD_RINDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_RINDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_ENABLED, "false");
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_CONFIG, "");
    conf.set("partitioner.category.shardedTables", BalancedShardPartitioner.class.getName());
    conf.set("partitioner.category.member." + TableName.SHARD, "shardedTables");
}
 
Example 6
Source File: DownSamplerTest.java    From ambari-metrics with Apache License 2.0 5 votes vote down vote up
@Ignore
@Test
public void testPrepareTopNDownSamplingStatement() throws Exception {
  Configuration configuration = new Configuration();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "3");

  Map<String, String> conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);

  TopNDownSampler topNDownSampler = TopNDownSampler.fromConfig(conf);
  List<String> stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD_UUID");
  Assert.assertEquals(stmts.size(),2);
  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  configuration.clear();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "4");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.function", "sum");
  conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
  topNDownSampler = TopNDownSampler.fromConfig(conf);
  stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_AGGREGATE_MINUTE_UUID");
  Assert.assertEquals(stmts.size(),1);

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "SUM(METRIC_SUM), 1, SUM(METRIC_SUM), SUM(METRIC_SUM) FROM METRIC_AGGREGATE_MINUTE_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4"));
}
 
Example 7
Source File: S3CredentialsTest.java    From mr4c with Apache License 2.0 5 votes vote down vote up
@Test public void testConfigurationUpdate() throws Exception {
	S3Credentials cred1 = buildS3Credentials1();
	Configuration conf = new Configuration();
	conf.clear();
	cred1.applyTo(conf);
	S3Credentials cred2 = S3Credentials.extractFrom(conf);
	assertEquals(cred1, cred2);
}
 
Example 8
Source File: TestHQuorumPeer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test public void testShouldAssignDefaultZookeeperClientPort() {
  Configuration config = HBaseConfiguration.create();
  config.clear();
  Properties p = ZKConfig.makeZKProps(config);
  assertNotNull(p);
  assertEquals(2181, p.get("clientPort"));
}
 
Example 9
Source File: TestTezUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Test
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 10
Source File: TestTezUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Test
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  byte[] bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 11
Source File: TestTezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout=2000)
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 12
Source File: TestTezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout=20000)
public void testByteStringToAndFromLargeConf() throws IOException {
  Configuration conf = getConf();
  int largeSizeMinimum = 64 * 1024 * 1024;
  final String alphaString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
  int largeSize = (largeSizeMinimum + alphaString.length() - 1) / alphaString.length();

  largeSize *= alphaString.length();
  assertTrue(largeSize >= alphaString.length());
  StringBuilder sb = new StringBuilder(largeSize);

  while (sb.length() < largeSize) {
    sb.append(alphaString);
  }

  String largeValue = sb.toString();
  Assert.assertEquals(largeSize, largeValue.length());
  conf.set("testLargeValue", largeValue);
  Assert.assertEquals(conf.size(), 7);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 7);
  checkConf(conf);
  Assert.assertEquals(conf.get("testLargeValue"), largeValue);
}
 
Example 13
Source File: TestTezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout=2000)
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  UserPayload bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 
Example 14
Source File: HBaseMultiClusterConfigUtil.java    From HBase.MCC with Apache License 2.0 4 votes vote down vote up
public static Configuration combineConfigurations(Configuration primary, Map<String, Configuration> failovers ) {

    Configuration resultingConfig = new Configuration();
    resultingConfig.clear();

    boolean isFirst = true;
    StringBuilder failOverClusterNames = new StringBuilder();


    Iterator<Entry<String, String>> primaryIt =  primary.iterator();

    while(primaryIt.hasNext()) {
      Entry<String, String> primaryKeyValue = primaryIt.next();
      resultingConfig.set(primaryKeyValue.getKey().replace('_', '.'), primaryKeyValue.getValue());
    }


    for (Entry<String, Configuration> failover: failovers.entrySet()) {
      if (isFirst) {
        isFirst = false;
      } else {
        failOverClusterNames.append(",");
      }
      failOverClusterNames.append(failover.getKey());

      Configuration failureConfig = failover.getValue();

      Iterator<Entry<String, String>> it = failureConfig.iterator();
      while (it.hasNext()) {
        Entry<String, String> keyValue = it.next();

        LOG.info(" -- Looking at : " + keyValue.getKey() + "=" + keyValue.getValue());

        String configKey = keyValue.getKey().replace('_', '.');

        if (configKey.startsWith("hbase.")) {
          resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTER_CONFIG + "." + failover.getKey() + "." + configKey , keyValue.getValue());
          LOG.info(" - Porting config: " + configKey + "=" + keyValue.getValue());
        }
      }
    }

    resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG, failOverClusterNames.toString());

    return resultingConfig;
  }