Java Code Examples for org.apache.hadoop.conf.Configuration#clear()

The following examples show how to use org.apache.hadoop.conf.Configuration#clear() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MapReduceExecutableTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}

Example 2

Source File: TestHdfsTarget.java From datacollector with Apache License 2.0

6 votes

@Test
public void testOnlyConfDirectory() throws Exception {
  // Create custom core-site.xml
  Configuration configuration = new Configuration();
  configuration.clear();
  configuration.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "file:///");
  FileOutputStream configOut = FileUtils.openOutputStream(new File(getTestDir() + "/conf-dir/core-site.xml"));
  configuration.writeXml(configOut);
  configOut.close();

  HdfsTarget hdfsTarget = HdfsTargetUtil.newBuilder()
    .hdfsUri("")
    .hdfsConfDir(getTestDir() + "/conf-dir/")
    .build();

  TargetRunner runner = new TargetRunner.Builder(HdfsDTarget.class, hdfsTarget)
    .setOnRecordError(OnRecordError.STOP_PIPELINE)
    .build();

  runner.runInit();

  // The configuration object should have the FS config from core-site.xml
  Assert.assertEquals("file:///", hdfsTarget.getHdfsConfiguration().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));

  runner.runDestroy();
}

Example 3

Source File: MapReduceExecutableTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}

Example 4

Source File: AbstractAccumuloStorage.java From spork with Apache License 2.0

6 votes

/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf,
        Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}

Example 5

Source File: MixedGeoAndGeoWaveTest.java From datawave with Apache License 2.0

5 votes

public static void setupConfiguration(Configuration conf) {
    conf.clear();
    
    conf.set(DATA_TYPE_NAME + BaseIngestHelper.INDEX_FIELDS, GEO_FIELD + "," + POINT_FIELD + "," + POLY_POINT_FIELD);
    conf.set(DATA_TYPE_NAME + "." + GEO_FIELD + BaseIngestHelper.FIELD_TYPE, GeoType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    conf.set(DATA_TYPE_NAME + "." + POLY_POINT_FIELD + BaseIngestHelper.FIELD_TYPE, PointType.class.getName());
    
    conf.set(DATA_TYPE_NAME + DataTypeHelper.Properties.INGEST_POLICY_ENFORCER_CLASS, IngestPolicyEnforcer.NoOpIngestPolicyEnforcer.class.getName());
    conf.set(DataTypeHelper.Properties.DATA_NAME, DATA_TYPE_NAME);
    conf.set(TypeRegistry.INGEST_DATA_TYPES, DATA_TYPE_NAME);
    conf.set(DATA_TYPE_NAME + TypeRegistry.INGEST_HELPER, INGEST_HELPER_CLASS);
    
    conf.set(ShardedDataTypeHandler.METADATA_TABLE_NAME, TableName.METADATA);
    conf.set(ShardedDataTypeHandler.NUM_SHARDS, Integer.toString(NUM_SHARDS));
    conf.set(ShardedDataTypeHandler.SHARDED_TNAMES, TableName.SHARD + "," + TableName.ERROR_SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_TNAME, TableName.SHARD);
    conf.set(ShardedDataTypeHandler.SHARD_LPRIORITY, "30");
    conf.set(TableName.SHARD + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_TNAME, TableName.SHARD_INDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_INDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_TNAME, TableName.SHARD_RINDEX);
    conf.set(ShardedDataTypeHandler.SHARD_GRIDX_LPRIORITY, "30");
    conf.set(TableName.SHARD_RINDEX + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX, ShardTableConfigHelper.class.getName());
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_ENABLED, "false");
    conf.set(ShardTableConfigHelper.MARKINGS_SETUP_ITERATOR_CONFIG, "");
    conf.set("partitioner.category.shardedTables", BalancedShardPartitioner.class.getName());
    conf.set("partitioner.category.member." + TableName.SHARD, "shardedTables");
}

Example 6

Source File: DownSamplerTest.java From ambari-metrics with Apache License 2.0

5 votes

@Ignore
@Test
public void testPrepareTopNDownSamplingStatement() throws Exception {
  Configuration configuration = new Configuration();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1,pattern2");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "3");

  Map<String, String> conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);

  TopNDownSampler topNDownSampler = TopNDownSampler.fromConfig(conf);
  List<String> stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_RECORD_UUID");
  Assert.assertEquals(stmts.size(),2);
  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "MAX(METRIC_MAX), 1, MAX(METRIC_MAX), MAX(METRIC_MAX) FROM METRIC_RECORD_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern2' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, HOSTNAME, APP_ID, INSTANCE_ID, UNITS ORDER BY MAX(METRIC_MAX) DESC LIMIT 3"));

  configuration.clear();
  configuration.setIfUnset("timeline.metrics.downsampler.topn.metric.patterns", "pattern1");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.value", "4");
  configuration.setIfUnset("timeline.metrics.downsampler.topn.function", "sum");
  conf = configuration.getValByRegex(DownSamplerUtils.downSamplerConfigPrefix);
  topNDownSampler = TopNDownSampler.fromConfig(conf);
  stmts = topNDownSampler.prepareDownSamplingStatement(14000000l, 14100000l, "METRIC_AGGREGATE_MINUTE_UUID");
  Assert.assertEquals(stmts.size(),1);

  Assert.assertTrue(stmts.contains("SELECT METRIC_NAME, APP_ID, INSTANCE_ID, 14100000 AS SERVER_TIME, UNITS, " +
    "SUM(METRIC_SUM), 1, SUM(METRIC_SUM), SUM(METRIC_SUM) FROM METRIC_AGGREGATE_MINUTE_UUID WHERE " +
    "METRIC_NAME LIKE 'pattern1' AND SERVER_TIME > 14000000 AND SERVER_TIME <= 14100000 " +
    "GROUP BY METRIC_NAME, APP_ID, INSTANCE_ID, UNITS ORDER BY SUM(METRIC_SUM) DESC LIMIT 4"));
}

Example 7

Source File: S3CredentialsTest.java From mr4c with Apache License 2.0

5 votes

@Test public void testConfigurationUpdate() throws Exception {
	S3Credentials cred1 = buildS3Credentials1();
	Configuration conf = new Configuration();
	conf.clear();
	cred1.applyTo(conf);
	S3Credentials cred2 = S3Credentials.extractFrom(conf);
	assertEquals(cred1, cred2);
}

Example 8

Source File: TestHQuorumPeer.java From hbase with Apache License 2.0

5 votes

@Test public void testShouldAssignDefaultZookeeperClientPort() {
  Configuration config = HBaseConfiguration.create();
  config.clear();
  Properties p = ZKConfig.makeZKProps(config);
  assertNotNull(p);
  assertEquals(2181, p.get("clientPort"));
}

Example 9

Source File: TestTezUtils.java From incubator-tez with Apache License 2.0

5 votes

@Test
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}

Example 10

Source File: TestTezUtils.java From incubator-tez with Apache License 2.0

5 votes

@Test
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  byte[] bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}

Example 11

Source File: TestTezUtils.java From tez with Apache License 2.0

5 votes

@Test (timeout=2000)
public void testByteStringToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}

Example 12

Source File: TestTezUtils.java From tez with Apache License 2.0

5 votes

@Test (timeout=20000)
public void testByteStringToAndFromLargeConf() throws IOException {
  Configuration conf = getConf();
  int largeSizeMinimum = 64 * 1024 * 1024;
  final String alphaString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
  int largeSize = (largeSizeMinimum + alphaString.length() - 1) / alphaString.length();

  largeSize *= alphaString.length();
  assertTrue(largeSize >= alphaString.length());
  StringBuilder sb = new StringBuilder(largeSize);

  while (sb.length() < largeSize) {
    sb.append(alphaString);
  }

  String largeValue = sb.toString();
  Assert.assertEquals(largeSize, largeValue.length());
  conf.set("testLargeValue", largeValue);
  Assert.assertEquals(conf.size(), 7);
  ByteString bsConf = TezUtils.createByteStringFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromByteString(bsConf);
  Assert.assertEquals(conf.size(), 7);
  checkConf(conf);
  Assert.assertEquals(conf.get("testLargeValue"), largeValue);
}

Example 13

Source File: TestTezUtils.java From tez with Apache License 2.0

5 votes

@Test (timeout=2000)
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  UserPayload bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}

Example 14

Source File: HBaseMultiClusterConfigUtil.java From HBase.MCC with Apache License 2.0

4 votes

public static Configuration combineConfigurations(Configuration primary, Map<String, Configuration> failovers ) {

    Configuration resultingConfig = new Configuration();
    resultingConfig.clear();

    boolean isFirst = true;
    StringBuilder failOverClusterNames = new StringBuilder();


    Iterator<Entry<String, String>> primaryIt =  primary.iterator();

    while(primaryIt.hasNext()) {
      Entry<String, String> primaryKeyValue = primaryIt.next();
      resultingConfig.set(primaryKeyValue.getKey().replace('_', '.'), primaryKeyValue.getValue());
    }


    for (Entry<String, Configuration> failover: failovers.entrySet()) {
      if (isFirst) {
        isFirst = false;
      } else {
        failOverClusterNames.append(",");
      }
      failOverClusterNames.append(failover.getKey());

      Configuration failureConfig = failover.getValue();

      Iterator<Entry<String, String>> it = failureConfig.iterator();
      while (it.hasNext()) {
        Entry<String, String> keyValue = it.next();

        LOG.info(" -- Looking at : " + keyValue.getKey() + "=" + keyValue.getValue());

        String configKey = keyValue.getKey().replace('_', '.');

        if (configKey.startsWith("hbase.")) {
          resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTER_CONFIG + "." + failover.getKey() + "." + configKey , keyValue.getValue());
          LOG.info(" - Porting config: " + configKey + "=" + keyValue.getValue());
        }
      }
    }

    resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG, failOverClusterNames.toString());

    return resultingConfig;
  }