Java Code Examples for org.apache.hadoop.conf.Configuration#setFloat()

@Test
public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
      HeapMemoryTuner.class);
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  CustomHeapMemoryTuner.memstoreSize = 0.78f;
  CustomHeapMemoryTuner.blockCacheSize = 0.02f;
  Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
  // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
  // upper bound. Same with block cache as 10% is the lower bound.
  assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
  assertHeapSpace(0.1f, blockCache.maxSize);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  Configuration conf = HTU.getConfiguration();
  conf.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); // less number of retries is needed
  conf.setInt(HConstants.HBASE_CLIENT_SERVERSIDE_RETRIES_MULTIPLIER, 1);

  HTU.startMiniCluster(NB_SERVERS);
}
 

@Test
public void testOverrideYarnDiskHealthCheck() throws IOException {
  MiniTezCluster tezMiniCluster = new MiniTezCluster(TestMiniTezCluster.class.getName(), 1, 1, 1);
  tezMiniCluster.init(new Configuration());
  tezMiniCluster.start();

  // overrides if not set
  Assert.assertEquals(99.0, tezMiniCluster.getConfig()
      .getFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, -1), 0.00001);

  tezMiniCluster.close();

  tezMiniCluster = new MiniTezCluster(TestMiniTezCluster.class.getName(), 1, 1, 1);
  Configuration conf = new Configuration();
  conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 50);
  tezMiniCluster.init(conf);
  tezMiniCluster.start();

  // respects provided non-default value
  Assert.assertEquals(50.0, tezMiniCluster.getConfig()
      .getFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, -1), 0.00001);

  tezMiniCluster.close();
}
 

public static void setUpTestBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
  if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
    RSGroupUtil.enableRSGroup(conf);
  }
  if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
        CPMasterObserver.class.getName());
  } else {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
  }

  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
    NUM_SLAVES_BASE - 1);
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setInt("hbase.rpc.timeout", 100000);

  TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
  initialize();
}
 

@Test
public void testL2CacheWithInvalidBucketSize() {
  Configuration c = new Configuration(this.conf);
  c.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
  c.set(BlockCacheFactory.BUCKET_CACHE_BUCKETS_KEY, "256,512,1024,2048,4000,4096");
  c.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 1024);
  try {
    BlockCacheFactory.createBlockCache(c);
    fail("Should throw IllegalArgumentException when passing illegal value for bucket size");
  } catch (IllegalArgumentException e) {
  }
}
 

@Before
public void setUp() throws Exception {
  conf = new Configuration();
  // populate repl queues on standby (in safe mode)
  conf.setFloat("dfs.namenode.replqueue.threshold-pct", 0f);
  conf.setLong("fs.avatar.standbyfs.initinterval", 1000);
  conf.setLong("fs.avatar.standbyfs.checkinterval", 1000);
}
 

public BulkLoadHFilesTool(Configuration conf) {
  // make a copy, just to be sure we're not overriding someone else's config
  super(new Configuration(conf));
  // disable blockcache for tool invocation, see HBASE-10500
  conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
  userProvider = UserProvider.instantiate(conf);
  fsDelegationToken = new FsDelegationToken(userProvider, "renewer");
  assignSeqIds = conf.getBoolean(ASSIGN_SEQ_IDS, true);
  maxFilesPerRegionPerFamily = conf.getInt(MAX_FILES_PER_REGION_PER_FAMILY, 32);
  nrThreads = conf.getInt("hbase.loadincremental.threads.max",
    Runtime.getRuntime().availableProcessors());
  bulkLoadByFamily = conf.getBoolean(BULK_LOAD_HFILES_BY_FAMILY, false);
}
 

@Test
public void testCanTolerateBadRecords() throws Exception {
  Configuration conf = new Configuration();
  conf.setFloat(UnmaterializableRecordCounter.BAD_RECORD_THRESHOLD_CONF_KEY, 0.1f);

  List<StructWithUnionV2> expected = new ArrayList<StructWithUnionV2>();

  readFile(writeFileWithCorruptRecords(4, expected), conf, "testCanTolerateBadRecords");
  assertEquals(200, ReadMapper.records.size());
  assertEqualsExcepted(expected, ReadMapper.records);
}
 

public Configuration createConfiguration() {
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
      1024);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
  conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
  return conf;
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = new Configuration();
  conf.setInt("dfs.block.size", 1024);
  conf.setFloat("dfs.safemode.threshold.pct", 1.5f);
  cluster = new MiniDFSCluster(conf, 1, true, null, false);
  cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  fs = cluster.getFileSystem();
}
 

/**
 * Sets default conf.
 */
@Before
public void setDefaultConf() {
  try {
    defaultConf = new Configuration(false);
    defaultConf.set("fs.defaultFS", "file:///");
    defaultConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    defaultConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 1);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 1);
    localFs = FileSystem.getLocal(defaultConf);
    String stagingDir =
        "target" + Path.SEPARATOR + TestSpeculation.class.getName()
            + "-tmpDir";
    defaultConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir);
    defaultConf.setClass(TezConfiguration.TEZ_AM_TASK_ESTIMATOR_CLASS,
        estimatorClass,
        TaskRuntimeEstimator.class);
    defaultConf.setInt(TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, 20);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, 0.2);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, 0.25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, 25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, 50);
    defaultConf.setInt(TezConfiguration.TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS, 2);
  } catch (IOException e) {
    throw new RuntimeException("init failure", e);
  }
}
 

@Test
public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
  HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
      new MemstoreFlusherStub(0), new RegionServerStub(conf),
      regionServerAccounting);
  assertFalse(manager.isTunerOn());
}
 

@Test (timeout=2000)
public void tetCommonConf() {

  Configuration fromConf = new Configuration(false);
  fromConf.set("test.conf.key.1", "confkey1");
  fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false);
  fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f);
  fromConf.set("io.shouldExist", "io");
  Configuration fromConfUnfiltered = new Configuration(false);
  fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
  Map<String, String> additionalConfs = new HashMap<String, String>();
  additionalConfs.put("test.key.2", "key2");
  additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
  additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f");
  additionalConfs.put("file.shouldExist", "file");

  UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig
      .newBuilder("KEY",
          "VALUE")
      .setAdditionalConfiguration("fs.shouldExist", "fs")
      .setAdditionalConfiguration("test.key.1", "key1")
      .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333")
      .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
      .setAdditionalConfiguration(additionalConfs)
      .setFromConfiguration(fromConf)
      .setFromConfigurationUnfiltered(fromConfUnfiltered);

  UnorderedKVEdgeConfig configuration = builder.build();

  UnorderedKVOutputConfig rebuiltOutput =
      new UnorderedKVOutputConfig();
  rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
  UnorderedKVInputConfig rebuiltInput =
      new UnorderedKVInputConfig();
  rebuiltInput.fromUserPayload(configuration.getInputPayload());

  Configuration outputConf = rebuiltOutput.conf;
  Configuration inputConf = rebuiltInput.conf;

  assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
  assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT));
  assertEquals("io", outputConf.get("io.shouldExist"));
  assertEquals("file", outputConf.get("file.shouldExist"));
  assertEquals("fs", outputConf.get("fs.shouldExist"));

  assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));

  assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true));
  assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertEquals(0.11f,
      inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.001f);
  assertEquals(0.22f,
      inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f);
  assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f),
      0.001f);
  assertEquals("io", inputConf.get("io.shouldExist"));
  assertEquals("file", inputConf.get("file.shouldExist"));
  assertEquals("fs", inputConf.get("fs.shouldExist"));

  assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
}
 

@Test
public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
  MemstoreFlusherStub memStoreFlusher =
      new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  // Empty memstore and but nearly filled block cache
  blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
  regionServerAccounting.setTestMemstoreSize(0);
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), regionServerAccounting);
  long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  // do some offheap flushes also. So there should be decrease in memstore but
  // not as that when we don't have offheap flushes
  memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
  oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  oldBlockCacheSize = blockCache.maxSize;
  // Do some more evictions before the next run of HeapMemoryTuner
  blockCache.evictBlock(null);
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
}
 

public static void setFloatVar(Configuration conf, ConfVars var, float val) {
  conf.setFloat(var.varname, val);
}
 

@Test
public void tetCommonConf() {

  Configuration fromConf = new Configuration(false);
  fromConf.set("test.conf.key.1", "confkey1");
  fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 3);
  fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f);
  fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 123);
  fromConf.set("io.shouldExist", "io");
  Map<String, String> additionalConfs = new HashMap<String, String>();
  additionalConfs.put("test.key.2", "key2");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f");
  additionalConfs.put(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter");
  additionalConfs.put("file.shouldExist", "file");

  OrderedPartitionedKVEdgeConfigurer.Builder builder = OrderedPartitionedKVEdgeConfigurer
      .newBuilder("KEY", "VALUE", "PARTITIONER", null)
      .setAdditionalConfiguration("fs.shouldExist", "fs")
      .setAdditionalConfiguration("test.key.1", "key1")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f")
      .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333")
      .setAdditionalConfiguration(additionalConfs)
      .setFromConfiguration(fromConf);

  OrderedPartitionedKVEdgeConfigurer configuration = builder.build();

  byte[] outputBytes = configuration.getOutputPayload();
  byte[] inputBytes = configuration.getInputPayload();

  OnFileSortedOutputConfiguration rebuiltOutput = new OnFileSortedOutputConfiguration();
  rebuiltOutput.fromByteArray(outputBytes);
  ShuffledMergedInputConfiguration rebuiltInput = new ShuffledMergedInputConfiguration();
  rebuiltInput.fromByteArray(inputBytes);

  Configuration outputConf = rebuiltOutput.conf;
  Configuration inputConf = rebuiltInput.conf;

  assertEquals(3, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(2222, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT));
  assertEquals(123, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 0));
  assertEquals("CustomSorter", outputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS));
  assertEquals(3333,
      outputConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
  assertEquals("io", outputConf.get("io.shouldExist"));
  assertEquals("file", outputConf.get("file.shouldExist"));
  assertEquals("fs", outputConf.get("fs.shouldExist"));


  assertEquals(3, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(2222, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0));
  assertEquals(0.11f,
      inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f);
  assertEquals(0.22f,
      inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f);
  assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f),
      0.001f);
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB));
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS));
  assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES));
  assertEquals("io", inputConf.get("io.shouldExist"));
  assertEquals("file", inputConf.get("file.shouldExist"));
  assertEquals("fs", inputConf.get("fs.shouldExist"));

}
 

@Test
public void testInMemAllocation() throws IOException {
  String localDirs = "/tmp/" + this.getClass().getName();
  Configuration conf = new Configuration();
  
  long jvmMax = Runtime.getRuntime().maxMemory();
  LOG.info("jvmMax: " + jvmMax);
  
  float bufferPercent = 0.1f;
  conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, bufferPercent);
  conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f);
  conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
  
  long inMemThreshold = (long) (bufferPercent * jvmMax);
  LOG.info("InMemThreshold: " + inMemThreshold);

  SimpleFetchedInputAllocator inputManager = new SimpleFetchedInputAllocator(UUID.randomUUID().toString(),
      conf, Runtime.getRuntime().maxMemory(), inMemThreshold);

  long requestSize = (long) (0.4f * inMemThreshold);
  long compressedSize = 1l;
  LOG.info("RequestSize: " + requestSize);
  
  FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi1.getType());
  
  
  FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi2.getType());
  
  
  // Over limit by this point. Next reserve should give back a DISK allocation
  FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1));
  assertEquals(FetchedInput.Type.DISK, fi3.getType());
  
  
  // Freed one memory allocation. Next should be mem again.
  fi1.abort();
  fi1.free();
  FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
  assertEquals(FetchedInput.Type.MEMORY, fi4.getType());
  
  // Freed one disk allocation. Next sould be disk again (no mem freed)
  fi3.abort();
  fi3.free();
  FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1));
  assertEquals(FetchedInput.Type.DISK, fi5.getType());
}
 

/**
 * Set the map output data compression ratio in the given configuration.
 */
static void setMapOutputCompressionEmulationRatio(Configuration conf, 
                                                  float ratio) {
  conf.setFloat(GRIDMIX_MAP_OUTPUT_COMPRESSION_RATIO, ratio);
}
 

@Test
public void testScanQueues() throws Exception {
  Configuration schedConf = HBaseConfiguration.create();
  schedConf.setFloat(RpcExecutor.CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 1.0f);
  schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_READ_SHARE_CONF_KEY, 0.7f);
  schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_SCAN_SHARE_CONF_KEY, 0.5f);

  PriorityFunction priority = mock(PriorityFunction.class);
  when(priority.getPriority(any(), any(), any())).thenReturn(HConstants.NORMAL_QOS);

  RpcScheduler scheduler = new SimpleRpcScheduler(schedConf, 3, 1, 1, priority,
                                                  HConstants.QOS_THRESHOLD);
  try {
    scheduler.start();

    CallRunner putCallTask = mock(CallRunner.class);
    ServerCall putCall = mock(ServerCall.class);
    putCall.param = RequestConverter.buildMutateRequest(
        Bytes.toBytes("abc"), new Put(Bytes.toBytes("row")));
    RequestHeader putHead = RequestHeader.newBuilder().setMethodName("mutate").build();
    when(putCallTask.getRpcCall()).thenReturn(putCall);
    when(putCall.getHeader()).thenReturn(putHead);
    when(putCall.getParam()).thenReturn(putCall.param);

    CallRunner getCallTask = mock(CallRunner.class);
    ServerCall getCall = mock(ServerCall.class);
    RequestHeader getHead = RequestHeader.newBuilder().setMethodName("get").build();
    when(getCallTask.getRpcCall()).thenReturn(getCall);
    when(getCall.getHeader()).thenReturn(getHead);

    CallRunner scanCallTask = mock(CallRunner.class);
    ServerCall scanCall = mock(ServerCall.class);
    scanCall.param = ScanRequest.newBuilder().build();
    RequestHeader scanHead = RequestHeader.newBuilder().setMethodName("scan").build();
    when(scanCallTask.getRpcCall()).thenReturn(scanCall);
    when(scanCall.getHeader()).thenReturn(scanHead);
    when(scanCall.getParam()).thenReturn(scanCall.param);

    ArrayList<Integer> work = new ArrayList<>();
    doAnswerTaskExecution(putCallTask, work, 1, 1000);
    doAnswerTaskExecution(getCallTask, work, 2, 1000);
    doAnswerTaskExecution(scanCallTask, work, 3, 1000);

    // There are 3 queues: [puts], [gets], [scans]
    // so the calls will be interleaved
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(putCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(getCallTask);
    scheduler.dispatch(scanCallTask);
    scheduler.dispatch(scanCallTask);
    scheduler.dispatch(scanCallTask);

    while (work.size() < 6) {
      Thread.sleep(100);
    }

    for (int i = 0; i < work.size() - 2; i += 3) {
      assertNotEquals(work.get(i + 0), work.get(i + 1));
      assertNotEquals(work.get(i + 0), work.get(i + 2));
      assertNotEquals(work.get(i + 1), work.get(i + 2));
    }
  } finally {
    scheduler.stop();
  }
}
 

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException{
	//args[0] is the number of features each input has.
	num_features=Integer.parseInt(args[0]);
	++num_features;
	//args[1] is the value of alpha that you want to use.
	alpha=Float.parseFloat(args[1]);
	Configuration conf=new Configuration();
	FileSystem hdfs=FileSystem.get(conf);
	Float[] theta=new Float[num_features];
	//args[2] is the number of times you want to iterate over your training set.
	for(int i=0;i<Integer.parseInt(args[2]);i++){
		//for the first run
		if(i==0){
			for(int i1=0;i1<num_features;i1++){
				theta[i1]=(float) 0;
			}
		}
		//for the second run
		else{
			int iter=0;
			//args[4] is the output path for storing the theta values.
			BufferedReader br1 = new BufferedReader(new InputStreamReader(hdfs.open(new Path(args[4]))));
			String line1=null;
			while((line1=br1.readLine())!=null){
				String[] theta_line=line1.toString().split("\\\t");
				theta[iter]=Float.parseFloat(theta_line[1]);
				iter++;
			}
			br1.close();
		}
		if(hdfs.exists(new Path(args[4]))){
			hdfs.delete(new Path(args[4]),true);
		}
		hdfs.close();
		//alpha value initialisation
		conf.setFloat("alpha", alpha);
		//Theta Value Initialisation
		for(int j=0;j<num_features;j++){
			conf.setFloat("theta".concat(String.valueOf(j)), theta[j]);
		}
		Job job = new Job(conf,"Calculation of Theta");
		job.setJarByClass(Driver.class);
		//args[3] is the input path.
		FileInputFormat.setInputPaths(job, new Path(args[3]));
		FileOutputFormat.setOutputPath(job, new Path(args[4]));
		job.setMapperClass(thetaMAP.class);
		job.setReducerClass(thetaREDUCE.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(FloatWritable.class);
		job.waitForCompletion(true);
	}
}