Java Code Examples for org.apache.hadoop.conf.Configuration#setFloat()

The following examples show how to use org.apache.hadoop.conf.Configuration#setFloat() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: hbase   File: TestHeapMemoryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
      HeapMemoryTuner.class);
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  CustomHeapMemoryTuner.memstoreSize = 0.78f;
  CustomHeapMemoryTuner.blockCacheSize = 0.02f;
  Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
  // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
  // upper bound. Same with block cache as 10% is the lower bound.
  assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
  assertHeapSpace(0.1f, blockCache.maxSize);
}
 
Example 2
Source Project: hbase   File: TestRSGroupsBase.java    License: Apache License 2.0 6 votes vote down vote up
public static void setUpTestBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
  if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
    RSGroupUtil.enableRSGroup(conf);
  }
  if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
        CPMasterObserver.class.getName());
  } else {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
  }

  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
    NUM_SLAVES_BASE - 1);
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setInt("hbase.rpc.timeout", 100000);

  TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
  initialize();
}
 
Example 3
Source Project: tez   File: TestMiniTezCluster.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOverrideYarnDiskHealthCheck() throws IOException {
  MiniTezCluster tezMiniCluster = new MiniTezCluster(TestMiniTezCluster.class.getName(), 1, 1, 1);
  tezMiniCluster.init(new Configuration());
  tezMiniCluster.start();

  // overrides if not set
  Assert.assertEquals(99.0, tezMiniCluster.getConfig()
      .getFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, -1), 0.00001);

  tezMiniCluster.close();

  tezMiniCluster = new MiniTezCluster(TestMiniTezCluster.class.getName(), 1, 1, 1);
  Configuration conf = new Configuration();
  conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 50);
  tezMiniCluster.init(conf);
  tezMiniCluster.start();

  // respects provided non-default value
  Assert.assertEquals(50.0, tezMiniCluster.getConfig()
      .getFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, -1), 0.00001);

  tezMiniCluster.close();
}
 
Example 4
@BeforeClass
public static void beforeClass() throws Exception {
  Configuration conf = HTU.getConfiguration();
  conf.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); // less number of retries is needed
  conf.setInt(HConstants.HBASE_CLIENT_SERVERSIDE_RETRIES_MULTIPLIER, 1);

  HTU.startMiniCluster(NB_SERVERS);
}
 
Example 5
Source Project: hbase   File: TestHeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
  HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
      new MemstoreFlusherStub(0), new RegionServerStub(conf),
      regionServerAccounting);
  assertFalse(manager.isTunerOn());
}
 
Example 6
@Before
public void setUp() throws Exception {
  conf = new Configuration();
  // populate repl queues on standby (in safe mode)
  conf.setFloat("dfs.namenode.replqueue.threshold-pct", 0f);
  conf.setLong("fs.avatar.standbyfs.initinterval", 1000);
  conf.setLong("fs.avatar.standbyfs.checkinterval", 1000);
}
 
Example 7
Source Project: hbase   File: TestCacheConfig.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testL2CacheWithInvalidBucketSize() {
  Configuration c = new Configuration(this.conf);
  c.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
  c.set(BlockCacheFactory.BUCKET_CACHE_BUCKETS_KEY, "256,512,1024,2048,4000,4096");
  c.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 1024);
  try {
    BlockCacheFactory.createBlockCache(c);
    fail("Should throw IllegalArgumentException when passing illegal value for bucket size");
  } catch (IllegalArgumentException e) {
  }
}
 
Example 8
Source Project: hbase   File: BulkLoadHFilesTool.java    License: Apache License 2.0 5 votes vote down vote up
public BulkLoadHFilesTool(Configuration conf) {
  // make a copy, just to be sure we're not overriding someone else's config
  super(new Configuration(conf));
  // disable blockcache for tool invocation, see HBASE-10500
  conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
  userProvider = UserProvider.instantiate(conf);
  fsDelegationToken = new FsDelegationToken(userProvider, "renewer");
  assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, true);
  maxFilesPerRegionPerFamily = conf.getInt(MAX_FILES_PER_REGION_PER_FAMILY, 32);
  nrThreads = conf.getInt("hbase.loadincremental.threads.max",
    Runtime.getRuntime().availableProcessors());
  bulkLoadByFamily = conf.getBoolean(BULK_LOAD_HFILES_BY_FAMILY, false);
}
 
Example 9
@Test
public void testCanTolerateBadRecords() throws Exception {
  Configuration conf = new Configuration();
  conf.setFloat(UnmaterializableRecordCounter.BAD_RECORD_THRESHOLD_CONF_KEY, 0.1f);

  List<StructWithUnionV2> expected = new ArrayList<StructWithUnionV2>();

  readFile(writeFileWithCorruptRecords(4, expected), conf, "testCanTolerateBadRecords");
  assertEquals(200, ReadMapper.records.size());
  assertEqualsExcepted(expected, ReadMapper.records);
}
 
Example 10
Source Project: big-c   File: FairSchedulerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public Configuration createConfiguration() {
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
      1024);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
  conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
  return conf;
}
 
Example 11
Source Project: RDFS   File: TestRbwReportSafeMode.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = new Configuration();
  conf.setInt("dfs.block.size", 1024);
  conf.setFloat("dfs.safemode.threshold.pct", 1.5f);
  cluster = new MiniDFSCluster(conf, 1, true, null, false);
  cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  fs = cluster.getFileSystem();
}
 
Example 12
Source Project: tez   File: TestSpeculation.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets default conf.
 */
@Before
public void setDefaultConf() {
  try {
    defaultConf = new Configuration(false);
    defaultConf.set("fs.defaultFS", "file:///");
    defaultConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    defaultConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 1);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 1);
    localFs = FileSystem.getLocal(defaultConf);
    String stagingDir =
        "target" + Path.SEPARATOR + TestSpeculation.class.getName()
            + "-tmpDir";
    defaultConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir);
    defaultConf.setClass(TezConfiguration.TEZ_AM_TASK_ESTIMATOR_CLASS,
        estimatorClass,
        TaskRuntimeEstimator.class);
    defaultConf.setInt(TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, 20);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, 0.2);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, 0.25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, 25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, 50);
    defaultConf.setInt(TezConfiguration.TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS, 2);
  } catch (IOException e) {
    throw new RuntimeException("init failure", e);
  }
}
 
Example 13
@Test
public void testInMemAllocation() throws IOException {
  String localDirs = "/tmp/" + this.getClass().getName();
  Configuration conf = new Configuration();
  
  long jvmMax = Runtime.getRuntime().maxMemory();
  LOG.info("jvmMax: " + jvmMax);
  
  float bufferPercent = 0.1f;
  conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, bufferPercent);
  conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f);
  conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
  
  long inMemThreshold = (long) (bufferPercent * jvmMax);
  LOG.info("InMemThreshold: " + inMemThreshold);

  SimpleFetchedInputAllocator inputManager = new SimpleFetchedInputAllocator(UUID.randomUUID().toString(),
      conf, Runtime.getRuntime().maxMemory(), inMemThreshold);

  long requestSize = (long) (0.4f * inMemThreshold);
  long compressedSize = 1l;
  LOG.info("RequestSize: " + requestSize);
  
  FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi1.getType());
  
  
  FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi2.getType());
  
  
  // Over limit by this point. Next reserve should give back a DISK allocation
  FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1));
  assertEquals(FetchedInput.Type.DISK, fi3.getType());
  
  
  // Freed one memory allocation. Next should be mem again.
  fi1.abort();
  fi1.free();
  FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi4.getType());
  
  // Freed one disk allocation. Next sould be disk again (no mem freed)
  fi3.abort();
  fi3.free();
  FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
  assertEquals(FetchedInput.Type.DISK, fi5.getType());
}
 
Example 14
Source Project: tez   File: TestUnorderedKVEdgeConfig.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=2000)
public void tetCommonConf() {

  Configuration fromConf = new Configuration(false);
  fromConf.set("test.conf.key.1", "confkey1");
  fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false);
  fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f);
  fromConf.set("io.shouldExist", "io");
  Configuration fromConfUnfiltered = new Configuration(false);
  fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
  Map<String, String> additionalConfs = new HashMap<String, String>();
  additionalConfs.put("test.key.2", "key2");
  additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
  additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f");
  additionalConfs.put("file.shouldExist", "file");

  UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig
      .newBuilder("KEY",
          "VALUE")
      .setAdditionalConfiguration("fs.shouldExist", "fs")
      .setAdditionalConfiguration("test.key.1", "key1")
      .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333")
      .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
      .setAdditionalConfiguration(additionalConfs)
      .setFromConfiguration(fromConf)
      .setFromConfigurationUnfiltered(fromConfUnfiltered);

  UnorderedKVEdgeConfig configuration = builder.build();

  UnorderedKVOutputConfig rebuiltOutput =
      new UnorderedKVOutputConfig();
  rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
  UnorderedKVInputConfig rebuiltInput =
      new UnorderedKVInputConfig();
  rebuiltInput.fromUserPayload(configuration.getInputPayload());

  Configuration outputConf = rebuiltOutput.conf;
  Configuration inputConf = rebuiltInput.conf;

  assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
  assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT));
  assertEquals("io", outputConf.get("io.shouldExist"));
  assertEquals("file", outputConf.get("file.shouldExist"));
  assertEquals("fs", outputConf.get("fs.shouldExist"));

  assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));

  assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
  assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertEquals(0.11f,
      inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.001f);
  assertEquals(0.22f,
      inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f);
  assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f),
      0.001f);
  assertEquals("io", inputConf.get("io.shouldExist"));
  assertEquals("file", inputConf.get("file.shouldExist"));
  assertEquals("fs", inputConf.get("fs.shouldExist"));

  assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
}
 
Example 15
Source Project: hbase   File: TestHeapMemoryManager.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
  MemstoreFlusherStub memStoreFlusher =
      new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  // Empty memstore and but nearly filled block cache
  blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
  regionServerAccounting.setTestMemstoreSize(0);
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), regionServerAccounting);
  long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  // do some offheap flushes also. So there should be decrease in memstore but
  // not as that when we don't have offheap flushes
  memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
  oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  oldBlockCacheSize = blockCache.maxSize;
  // Do some more evictions before the next run of HeapMemoryTuner
  blockCache.evictBlock(null);
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
}
 
Example 16
Source Project: hive-dwrf   File: OrcConf.java    License: Apache License 2.0 4 votes vote down vote up
public static void setFloatVar(Configuration conf, ConfVars var, float val) {
  conf.setFloat(var.varname, val);
}
 
Example 17
@Test
public void tetCommonConf() {

  Configuration fromConf = new Configuration(false);
  fromConf.set("test.conf.key.1", "confkey1");
  fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 3);
  fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f);
  fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 123);
  fromConf.set("io.shouldExist", "io");
  Map<String, String> additionalConfs = new HashMap<String, String>();
  additionalConfs.put("test.key.2", "key2");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter");
  additionalConfs.put("file.shouldExist", "file");

  OrderedPartitionedKVEdgeConfigurer.Builder builder = OrderedPartitionedKVEdgeConfigurer
      .newBuilder("KEY", "VALUE", "PARTITIONER", null)
      .setAdditionalConfiguration("fs.shouldExist", "fs")
      .setAdditionalConfiguration("test.key.1", "key1")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333")
      .setAdditionalConfiguration(additionalConfs)
      .setFromConfiguration(fromConf);

  OrderedPartitionedKVEdgeConfigurer configuration = builder.build();

  byte[] outputBytes = configuration.getOutputPayload();
  byte[] inputBytes = configuration.getInputPayload();

  OnFileSortedOutputConfiguration rebuiltOutput = new OnFileSortedOutputConfiguration();
  rebuiltOutput.fromByteArray(outputBytes);
  ShuffledMergedInputConfiguration rebuiltInput = new ShuffledMergedInputConfiguration();
  rebuiltInput.fromByteArray(inputBytes);

  Configuration outputConf = rebuiltOutput.conf;
  Configuration inputConf = rebuiltInput.conf;

  assertEquals(3, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(2222, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT));
  assertEquals(123, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 0));
  assertEquals("CustomSorter", outputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS));
  assertEquals(3333,
      outputConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
  assertEquals("io", outputConf.get("io.shouldExist"));
  assertEquals("file", outputConf.get("file.shouldExist"));
  assertEquals("fs", outputConf.get("fs.shouldExist"));


  assertEquals(3, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(2222, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertEquals(0.11f,
      inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f);
  assertEquals(0.22f,
      inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f);
  assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f),
      0.001f);
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB));
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS));
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES));
  assertEquals("io", inputConf.get("io.shouldExist"));
  assertEquals("file", inputConf.get("file.shouldExist"));
  assertEquals("fs", inputConf.get("fs.shouldExist"));

}
 
Example 18
Source Project: hadoop   File: CompressionEmulationUtil.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Set the map output data compression ratio in the given configuration.
 */
static void setMapOutputCompressionEmulationRatio(Configuration conf, 
                                                  float ratio) {
  conf.setFloat(GRIDMIX_MAP_OUTPUT_COMPRESSION_RATIO, ratio);
}
 
Example 19
Source Project: hbase   File: TestSimpleRpcScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testScanQueues() throws Exception {
  Configuration schedConf = HBaseConfiguration.create();
  schedConf.setFloat(RpcExecutor.CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 1.0f);
  schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_READ_SHARE_CONF_KEY, 0.7f);
  schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_SCAN_SHARE_CONF_KEY, 0.5f);

  PriorityFunction priority = mock(PriorityFunction.class);
  when(priority.getPriority(any(), any(), any())).thenReturn(HConstants.NORMAL_QOS);

  RpcScheduler scheduler = new SimpleRpcScheduler(schedConf, 3, 1, 1, priority,
                                                  HConstants.QOS_THRESHOLD);
  try {
    scheduler.start();

    CallRunner putCallTask = mock(CallRunner.class);
    ServerCall putCall = mock(ServerCall.class);
    putCall.param = RequestConverter.buildMutateRequest(
        Bytes.toBytes("abc"), new Put(Bytes.toBytes("row")));
    RequestHeader putHead = RequestHeader.newBuilder().setMethodName("mutate").build();
    when(putCallTask.getRpcCall()).thenReturn(putCall);
    when(putCall.getHeader()).thenReturn(putHead);
    when(putCall.getParam()).thenReturn(putCall.param);

    CallRunner getCallTask = mock(CallRunner.class);
    ServerCall getCall = mock(ServerCall.class);
    RequestHeader getHead = RequestHeader.newBuilder().setMethodName("get").build();
    when(getCallTask.getRpcCall()).thenReturn(getCall);
    when(getCall.getHeader()).thenReturn(getHead);

    CallRunner scanCallTask = mock(CallRunner.class);
    ServerCall scanCall = mock(ServerCall.class);
    scanCall.param = ScanRequest.newBuilder().build();
    RequestHeader scanHead = RequestHeader.newBuilder().setMethodName("scan").build();
    when(scanCallTask.getRpcCall()).thenReturn(scanCall);
    when(scanCall.getHeader()).thenReturn(scanHead);
    when(scanCall.getParam()).thenReturn(scanCall.param);

    ArrayList<Integer> work = new ArrayList<>();
    doAnswerTaskExecution(putCallTask, work, 1, 1000);
    doAnswerTaskExecution(getCallTask, work, 2, 1000);
    doAnswerTaskExecution(scanCallTask, work, 3, 1000);

    // There are 3 queues: [puts], [gets], [scans]
    // so the calls will be interleaved
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(scanCallTask);
    scheduler.dispatch(scanCallTask);
    scheduler.dispatch(scanCallTask);

    while (work.size() < 6) {
      Thread.sleep(100);
    }

    for (int i = 0; i < work.size() - 2; i += 3) {
      assertNotEquals(work.get(i + 0), work.get(i + 1));
      assertNotEquals(work.get(i + 0), work.get(i + 2));
      assertNotEquals(work.get(i + 1), work.get(i + 2));
    }
  } finally {
    scheduler.stop();
  }
}
 
Example 20
Source Project: MLHadoop   File: Driver.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException{
	//args[0] is the number of features each input has.
	num_features=Integer.parseInt(args[0]);
	++num_features;
	//args[1] is the value of alpha that you want to use.
	alpha=Float.parseFloat(args[1]);
	Configuration conf=new Configuration();
	FileSystem hdfs=FileSystem.get(conf);
	Float[] theta=new Float[num_features];
	//args[2] is the number of times you want to iterate over your training set.
	for(int i=0;i<Integer.parseInt(args[2]);i++){
		//for the first run
		if(i==0){
			for(int i1=0;i1<num_features;i1++){
				theta[i1]=(float) 0;
			}
		}
		//for the second run
		else{
			int iter=0;
			//args[4] is the output path for storing the theta values.
			BufferedReader br1 = new BufferedReader(new InputStreamReader(hdfs.open(new Path(args[4]))));
			String line1=null;
			while((line1=br1.readLine())!=null){
				String[] theta_line=line1.toString().split("\\\t");
				theta[iter]=Float.parseFloat(theta_line[1]);
				iter++;
			}
			br1.close();
		}
		if(hdfs.exists(new Path(args[4]))){
			hdfs.delete(new Path(args[4]),true);
		}
		hdfs.close();
		//alpha value initialisation
		conf.setFloat("alpha", alpha);
		//Theta Value Initialisation
		for(int j=0;j<num_features;j++){
			conf.setFloat("theta".concat(String.valueOf(j)), theta[j]);
		}
		Job job = new Job(conf,"Calculation of Theta");
		job.setJarByClass(Driver.class);
		//args[3] is the input path.
		FileInputFormat.setInputPaths(job, new Path(args[3]));
		FileOutputFormat.setOutputPath(job, new Path(args[4]));
		job.setMapperClass(thetaMAP.class);
		job.setReducerClass(thetaREDUCE.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(FloatWritable.class);
		job.waitForCompletion(true);
	}
}