Java Code Examples for org.apache.hadoop.conf.Configuration#getDouble()

The following examples show how to use org.apache.hadoop.conf.Configuration#getDouble() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
public TimelineMetricSplitPointComputer(Configuration metricsConf,
                                        Configuration hbaseConf,
                                        TimelineMetricMetadataManager timelineMetricMetadataManager) {

  String componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    masterComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

 componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    slaveComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

  this.timelineMetricMetadataManager = timelineMetricMetadataManager;
  hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
  hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3);
  hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
}
 
Example 2
Source Project: hadoop   File: DefaultSpeculator.java    License: Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 3
Source Project: big-c   File: DefaultSpeculator.java    License: Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 4
Source Project: hbase   File: CompactingMemStore.java    License: Apache License 2.0 6 votes vote down vote up
private void initInmemoryFlushSize(Configuration conf) {
  double factor = 0;
  long memstoreFlushSize = getRegionServices().getMemStoreFlushSize();
  int numStores = getRegionServices().getNumStores();
  if (numStores <= 1) {
    // Family number might also be zero in some of our unit test case
    numStores = 1;
  }
  factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.0);
  if(factor != 0.0) {
    // multiply by a factor (the same factor for all index types)
    inmemoryFlushSize = (long) (factor * memstoreFlushSize) / numStores;
  } else {
    inmemoryFlushSize = IN_MEMORY_FLUSH_MULTIPLIER *
        conf.getLong(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT);
    inmemoryFlushSize -= ChunkCreator.SIZEOF_CHUNK_HEADER;
  }
}
 
Example 5
@Override
protected void configureForRegion(HRegion region) {
  super.configureForRegion(region);
  Configuration conf = getConf();
  TableDescriptor desc = region.getTableDescriptor();
  if (desc != null) {
    this.desiredMaxFileSize = desc.getMaxFileSize();
  }
  if (this.desiredMaxFileSize <= 0) {
    this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
      HConstants.DEFAULT_MAX_FILE_SIZE);
  }
  double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
  this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
  long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
  // make sure the long value won't overflow with jitter
  if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) {
    this.desiredMaxFileSize = Long.MAX_VALUE;
  } else {
    this.desiredMaxFileSize += jitterValue;
  }
}
 
Example 6
Source Project: tez   File: LegacySpeculator.java    License: Apache License 2.0 6 votes vote down vote up
public LegacySpeculator
    (Configuration conf, TaskRuntimeEstimator estimator, Clock clock, Vertex vertex) {
  super(LegacySpeculator.class.getName());
  this.vertex = vertex;
  this.estimator = estimator;
  this.clock = clock;
  taskTimeout = conf.getLong(
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT,
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT);
  soonestRetryAfterNoSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT);
  soonestRetryAfterSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT);
  proportionRunningTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT);
  proportionTotalTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT);
  minimumAllowedSpeculativeTasks = conf.getInt(
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS,
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT);
}
 
Example 7
@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();

    verbose = configuration.getBoolean(LouvainMR.VERBOSE,false);
    nb_pass = configuration.getInt(LouvainMR.NB_PASS, 0);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    outpath = configuration.get(LouvainMR.OUT_PATH);

    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);


    super.setup(context);

}
 
Example 8
Source Project: hadoop   File: DecayRpcScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 
Example 9
Source Project: big-c   File: DecayRpcScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 
Example 10
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    OrcNewSplit orcNewSplit = (OrcNewSplit) inputSplit;
    Configuration configuration = taskAttemptContext.getConfiguration();
    double maxMergeDistance = configuration.getDouble(MAX_MERGE_DISTANCE,MAX_MERGE_DISTANCE_DEFAULT);
    double maxReadSize = configuration.getDouble(MAX_READ_SIZE,MAX_READ_SIZE_DEFAULT);
    double streamBufferSize = configuration.getDouble(STREAM_BUFFER_SIZE,STREAM_BUFFER_SIZE_DEFAULT);
    Path path = orcNewSplit.getPath();
    FileSystem fileSystem = FileSystem.get(path.toUri(),configuration);
    long size = fileSystem.getFileStatus(path).getLen();
    FSDataInputStream inputStream = fileSystem.open(path);
    rowStruct = getRowStruct(configuration);
    predicate = getSplicePredicate(configuration);
    List<Integer> partitions = getPartitionIds(configuration);
    List<Integer> columnIds = getColumnIds(configuration);



    List<String> values = null;
    try {
        values = Warehouse.getPartValuesFromPartName(((OrcNewSplit) inputSplit).getPath().toString());
    } catch (MetaException me) {
        throw new IOException(me);
    }
    OrcDataSource orcDataSource = new HdfsOrcDataSource(path.toString(), size, new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE),
            new DataSize(streamBufferSize, DataSize.Unit.MEGABYTE), inputStream);
    OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE));
    orcRecordReader =
        orcReader.createRecordReader(getColumnsAndTypes(columnIds, rowStruct),
                                     predicate, orcNewSplit.getStart(), orcNewSplit.getLength(),
                                     HIVE_STORAGE_TIME_ZONE, new AggregatedMemoryContext(),
                                     partitions, values);

}
 
Example 11
public AdaptiveMemStoreCompactionStrategy(Configuration conf, String cfName) {
  super(conf, cfName);
  compactionThreshold = conf.getDouble(ADAPTIVE_COMPACTION_THRESHOLD_KEY,
      ADAPTIVE_COMPACTION_THRESHOLD_DEFAULT);
  initialCompactionProbability = conf.getDouble(ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_KEY,
      ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_DEFAULT);
  resetStats();
}
 
Example 12
Source Project: hbase   File: RpcExecutor.java    License: Apache License 2.0 5 votes vote down vote up
public void onConfigurationChange(Configuration conf) {
  // update CoDel Scheduler tunables
  int codelTargetDelay = conf.getInt(CALL_QUEUE_CODEL_TARGET_DELAY,
    CALL_QUEUE_CODEL_DEFAULT_TARGET_DELAY);
  int codelInterval = conf.getInt(CALL_QUEUE_CODEL_INTERVAL, CALL_QUEUE_CODEL_DEFAULT_INTERVAL);
  double codelLifoThreshold = conf.getDouble(CALL_QUEUE_CODEL_LIFO_THRESHOLD,
    CALL_QUEUE_CODEL_DEFAULT_LIFO_THRESHOLD);

  for (BlockingQueue<CallRunner> queue : queues) {
    if (queue instanceof AdaptiveLifoCoDelCallQueue) {
      ((AdaptiveLifoCoDelCallQueue) queue).updateTunables(codelTargetDelay, codelInterval,
        codelLifoThreshold);
    }
  }
}
 
Example 13
Source Project: laser   File: LrIterationMapper.java    License: Apache License 2.0 5 votes vote down vote up
protected void setup(Context context) throws IOException,
		InterruptedException {
	Configuration conf = context.getConfiguration();
	regularizationFactor = conf.getDouble(
			"lr.iteration.regulariztion.factor",
			DEFAULT_REGULARIZATION_FACTOR);
	lbfgs = new QNMinimizer();
	lbfgs.setRobustOptions();
}
 
Example 14
@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();
    verbose = configuration.getBoolean(LouvainMR.VERBOSE, false);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    this.outpath = configuration.get(LouvainMR.OUT_PATH);
    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);

    super.setup(context);
}
 
Example 15
Source Project: hbase   File: HMaster.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Initializes the HMaster. The steps are as follows:
 * <p>
 * <ol>
 * <li>Initialize the local HRegionServer
 * <li>Start the ActiveMasterManager.
 * </ol>
 * <p>
 * Remaining steps of initialization occur in
 * {@link #finishActiveMasterInitialization(MonitoredTask)} after the master becomes the
 * active one.
 */
public HMaster(final Configuration conf) throws IOException {
  super(conf);
  TraceUtil.initTracer(conf);
  try {
    if (conf.getBoolean(MAINTENANCE_MODE, false)) {
      LOG.info("Detected {}=true via configuration.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else if (Boolean.getBoolean(MAINTENANCE_MODE)) {
      LOG.info("Detected {}=true via environment variables.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else {
      maintenanceMode = false;
    }
    this.rsFatals = new MemoryBoundedLogMessageBuffer(
        conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
    LOG.info("hbase.rootdir={}, hbase.cluster.distributed={}", getDataRootDir(),
        this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));

    // Disable usage of meta replicas in the master
    this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);

    decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this
    // config param for task trackers, but we can piggyback off of it.
    if (this.conf.get("mapreduce.task.attempt.id") == null) {
      this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
    }

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));

    // preload table descriptor at startup
    this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);

    this.maxBalancingTime = getMaxBalancingTime();
    this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
        HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);

    // Do we publish the status?

    boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
        HConstants.STATUS_PUBLISHED_DEFAULT);
    Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
        conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.Publisher.class);

    if (shouldPublish) {
      if (publisherClass == null) {
        LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
            " is not set - not publishing status");
      } else {
        clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
        LOG.debug("Created {}", this.clusterStatusPublisherChore);
        getChoreService().scheduleChore(clusterStatusPublisherChore);
      }
    }

    // Some unit tests don't need a cluster, so no zookeeper at all
    if (!conf.getBoolean("hbase.testing.nocluster", false)) {
      this.metaRegionLocationCache = new MetaRegionLocationCache(this.zooKeeper);
      this.activeMasterManager = createActiveMasterManager(zooKeeper, serverName, this);
    } else {
      this.metaRegionLocationCache = null;
      this.activeMasterManager = null;
    }
    cachedClusterId = new CachedClusterId(this, conf);
  } catch (Throwable t) {
    // Make sure we log the exception. HMaster is often started via reflection and the
    // cause of failed startup is lost.
    LOG.error("Failed construction of Master", t);
    throw t;
  }
}
 
Example 16
Source Project: hbase   File: LossyCounting.java    License: Apache License 2.0 4 votes vote down vote up
public LossyCounting(String name, Configuration conf, LossyCountingListener<T> listener) {
  this(name, conf.getDouble(HConstants.DEFAULT_LOSSY_COUNTING_ERROR_RATE, 0.02), listener);
}
 
Example 17
Source Project: hbase   File: FaultyMobStoreCompactor.java    License: Apache License 2.0 4 votes vote down vote up
public FaultyMobStoreCompactor(Configuration conf, HStore store) {
  super(conf, store);
  failureProb = conf.getDouble("hbase.mob.compaction.fault.probability", 0.1);
}
 
Example 18
public static double getLookupRatio(Configuration configuration) {
  return configuration.getDouble(BLUR_LOOKUP_RATIO_PER_SHARD, DEFAULT_LOOKUP_RATIO);
}
 
Example 19
Source Project: hbase   File: QuotaObserverChore.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY,
      QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT);
}
 
Example 20
Source Project: hbase   File: SpaceQuotaRefresherChore.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(POLICY_REFRESHER_CHORE_REPORT_PERCENT_KEY,
      POLICY_REFRESHER_CHORE_REPORT_PERCENT_DEFAULT);
}