Java Code Examples for org.apache.hadoop.conf.Configuration#getDouble()

The following examples show how to use org.apache.hadoop.conf.Configuration#getDouble() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DefaultSpeculator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 2
Source File: MapCommunity.java    From hadoop-louvain-community with Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();

    verbose = configuration.getBoolean(LouvainMR.VERBOSE,false);
    nb_pass = configuration.getInt(LouvainMR.NB_PASS, 0);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    outpath = configuration.get(LouvainMR.OUT_PATH);

    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);


    super.setup(context);

}
 
Example 3
Source File: LegacySpeculator.java    From tez with Apache License 2.0 6 votes vote down vote up
public LegacySpeculator
    (Configuration conf, TaskRuntimeEstimator estimator, Clock clock, Vertex vertex) {
  super(LegacySpeculator.class.getName());
  this.vertex = vertex;
  this.estimator = estimator;
  this.clock = clock;
  taskTimeout = conf.getLong(
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT,
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT);
  soonestRetryAfterNoSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT);
  soonestRetryAfterSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT);
  proportionRunningTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT);
  proportionTotalTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT);
  minimumAllowedSpeculativeTasks = conf.getInt(
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS,
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT);
}
 
Example 4
Source File: ConstantSizeRegionSplitPolicy.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected void configureForRegion(HRegion region) {
  super.configureForRegion(region);
  Configuration conf = getConf();
  TableDescriptor desc = region.getTableDescriptor();
  if (desc != null) {
    this.desiredMaxFileSize = desc.getMaxFileSize();
  }
  if (this.desiredMaxFileSize <= 0) {
    this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
      HConstants.DEFAULT_MAX_FILE_SIZE);
  }
  double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
  this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
  long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
  // make sure the long value won't overflow with jitter
  if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) {
    this.desiredMaxFileSize = Long.MAX_VALUE;
  } else {
    this.desiredMaxFileSize += jitterValue;
  }
}
 
Example 5
Source File: CompactingMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void initInmemoryFlushSize(Configuration conf) {
  double factor = 0;
  long memstoreFlushSize = getRegionServices().getMemStoreFlushSize();
  int numStores = getRegionServices().getNumStores();
  if (numStores <= 1) {
    // Family number might also be zero in some of our unit test case
    numStores = 1;
  }
  factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.0);
  if(factor != 0.0) {
    // multiply by a factor (the same factor for all index types)
    inmemoryFlushSize = (long) (factor * memstoreFlushSize) / numStores;
  } else {
    inmemoryFlushSize = IN_MEMORY_FLUSH_MULTIPLIER *
        conf.getLong(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT);
    inmemoryFlushSize -= ChunkCreator.SIZEOF_CHUNK_HEADER;
  }
}
 
Example 6
Source File: DefaultSpeculator.java    From big-c with Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 7
Source File: TimelineMetricSplitPointComputer.java    From ambari-metrics with Apache License 2.0 6 votes vote down vote up
public TimelineMetricSplitPointComputer(Configuration metricsConf,
                                        Configuration hbaseConf,
                                        TimelineMetricMetadataManager timelineMetricMetadataManager) {

  String componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    masterComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

 componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    slaveComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

  this.timelineMetricMetadataManager = timelineMetricMetadataManager;
  hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
  hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3);
  hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
}
 
Example 8
Source File: OrcMapreduceRecordReader.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    OrcNewSplit orcNewSplit = (OrcNewSplit) inputSplit;
    Configuration configuration = taskAttemptContext.getConfiguration();
    double maxMergeDistance = configuration.getDouble(MAX_MERGE_DISTANCE,MAX_MERGE_DISTANCE_DEFAULT);
    double maxReadSize = configuration.getDouble(MAX_READ_SIZE,MAX_READ_SIZE_DEFAULT);
    double streamBufferSize = configuration.getDouble(STREAM_BUFFER_SIZE,STREAM_BUFFER_SIZE_DEFAULT);
    Path path = orcNewSplit.getPath();
    FileSystem fileSystem = FileSystem.get(path.toUri(),configuration);
    long size = fileSystem.getFileStatus(path).getLen();
    FSDataInputStream inputStream = fileSystem.open(path);
    rowStruct = getRowStruct(configuration);
    predicate = getSplicePredicate(configuration);
    List<Integer> partitions = getPartitionIds(configuration);
    List<Integer> columnIds = getColumnIds(configuration);



    List<String> values = null;
    try {
        values = Warehouse.getPartValuesFromPartName(((OrcNewSplit) inputSplit).getPath().toString());
    } catch (MetaException me) {
        throw new IOException(me);
    }
    OrcDataSource orcDataSource = new HdfsOrcDataSource(path.toString(), size, new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE),
            new DataSize(streamBufferSize, DataSize.Unit.MEGABYTE), inputStream);
    OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE));
    orcRecordReader =
        orcReader.createRecordReader(getColumnsAndTypes(columnIds, rowStruct),
                                     predicate, orcNewSplit.getStart(), orcNewSplit.getLength(),
                                     HIVE_STORAGE_TIME_ZONE, new AggregatedMemoryContext(),
                                     partitions, values);

}
 
Example 9
Source File: DecayRpcScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 
Example 10
Source File: AdaptiveMemStoreCompactionStrategy.java    From hbase with Apache License 2.0 5 votes vote down vote up
public AdaptiveMemStoreCompactionStrategy(Configuration conf, String cfName) {
  super(conf, cfName);
  compactionThreshold = conf.getDouble(ADAPTIVE_COMPACTION_THRESHOLD_KEY,
      ADAPTIVE_COMPACTION_THRESHOLD_DEFAULT);
  initialCompactionProbability = conf.getDouble(ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_KEY,
      ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_DEFAULT);
  resetStats();
}
 
Example 11
Source File: DecayRpcScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 
Example 12
Source File: RpcExecutor.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void onConfigurationChange(Configuration conf) {
  // update CoDel Scheduler tunables
  int codelTargetDelay = conf.getInt(CALL_QUEUE_CODEL_TARGET_DELAY,
    CALL_QUEUE_CODEL_DEFAULT_TARGET_DELAY);
  int codelInterval = conf.getInt(CALL_QUEUE_CODEL_INTERVAL, CALL_QUEUE_CODEL_DEFAULT_INTERVAL);
  double codelLifoThreshold = conf.getDouble(CALL_QUEUE_CODEL_LIFO_THRESHOLD,
    CALL_QUEUE_CODEL_DEFAULT_LIFO_THRESHOLD);

  for (BlockingQueue<CallRunner> queue : queues) {
    if (queue instanceof AdaptiveLifoCoDelCallQueue) {
      ((AdaptiveLifoCoDelCallQueue) queue).updateTunables(codelTargetDelay, codelInterval,
        codelLifoThreshold);
    }
  }
}
 
Example 13
Source File: LrIterationMapper.java    From laser with Apache License 2.0 5 votes vote down vote up
protected void setup(Context context) throws IOException,
		InterruptedException {
	Configuration conf = context.getConfiguration();
	regularizationFactor = conf.getDouble(
			"lr.iteration.regulariztion.factor",
			DEFAULT_REGULARIZATION_FACTOR);
	lbfgs = new QNMinimizer();
	lbfgs.setRobustOptions();
}
 
Example 14
Source File: ReduceCommunity.java    From hadoop-louvain-community with Apache License 2.0 5 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();
    verbose = configuration.getBoolean(LouvainMR.VERBOSE, false);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    this.outpath = configuration.get(LouvainMR.OUT_PATH);
    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);

    super.setup(context);
}
 
Example 15
Source File: HMaster.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Initializes the HMaster. The steps are as follows:
 * <p>
 * <ol>
 * <li>Initialize the local HRegionServer
 * <li>Start the ActiveMasterManager.
 * </ol>
 * <p>
 * Remaining steps of initialization occur in
 * {@link #finishActiveMasterInitialization(MonitoredTask)} after the master becomes the
 * active one.
 */
public HMaster(final Configuration conf) throws IOException {
  super(conf);
  TraceUtil.initTracer(conf);
  try {
    if (conf.getBoolean(MAINTENANCE_MODE, false)) {
      LOG.info("Detected {}=true via configuration.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else if (Boolean.getBoolean(MAINTENANCE_MODE)) {
      LOG.info("Detected {}=true via environment variables.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else {
      maintenanceMode = false;
    }
    this.rsFatals = new MemoryBoundedLogMessageBuffer(
        conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
    LOG.info("hbase.rootdir={}, hbase.cluster.distributed={}", getDataRootDir(),
        this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));

    // Disable usage of meta replicas in the master
    this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);

    decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this
    // config param for task trackers, but we can piggyback off of it.
    if (this.conf.get("mapreduce.task.attempt.id") == null) {
      this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
    }

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));

    // preload table descriptor at startup
    this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);

    this.maxBalancingTime = getMaxBalancingTime();
    this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
        HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);

    // Do we publish the status?

    boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
        HConstants.STATUS_PUBLISHED_DEFAULT);
    Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
        conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.Publisher.class);

    if (shouldPublish) {
      if (publisherClass == null) {
        LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
            " is not set - not publishing status");
      } else {
        clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
        LOG.debug("Created {}", this.clusterStatusPublisherChore);
        getChoreService().scheduleChore(clusterStatusPublisherChore);
      }
    }

    // Some unit tests don't need a cluster, so no zookeeper at all
    if (!conf.getBoolean("hbase.testing.nocluster", false)) {
      this.metaRegionLocationCache = new MetaRegionLocationCache(this.zooKeeper);
      this.activeMasterManager = createActiveMasterManager(zooKeeper, serverName, this);
    } else {
      this.metaRegionLocationCache = null;
      this.activeMasterManager = null;
    }
    cachedClusterId = new CachedClusterId(this, conf);
  } catch (Throwable t) {
    // Make sure we log the exception. HMaster is often started via reflection and the
    // cause of failed startup is lost.
    LOG.error("Failed construction of Master", t);
    throw t;
  }
}
 
Example 16
Source File: LossyCounting.java    From hbase with Apache License 2.0 4 votes vote down vote up
public LossyCounting(String name, Configuration conf, LossyCountingListener<T> listener) {
  this(name, conf.getDouble(HConstants.DEFAULT_LOSSY_COUNTING_ERROR_RATE, 0.02), listener);
}
 
Example 17
Source File: FaultyMobStoreCompactor.java    From hbase with Apache License 2.0 4 votes vote down vote up
public FaultyMobStoreCompactor(Configuration conf, HStore store) {
  super(conf, store);
  failureProb = conf.getDouble("hbase.mob.compaction.fault.probability", 0.1);
}
 
Example 18
Source File: InputSplitPruneUtil.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up
public static double getLookupRatio(Configuration configuration) {
  return configuration.getDouble(BLUR_LOOKUP_RATIO_PER_SHARD, DEFAULT_LOOKUP_RATIO);
}
 
Example 19
Source File: QuotaObserverChore.java    From hbase with Apache License 2.0 2 votes vote down vote up
/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY,
      QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT);
}
 
Example 20
Source File: SpaceQuotaRefresherChore.java    From hbase with Apache License 2.0 2 votes vote down vote up
/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(POLICY_REFRESHER_CHORE_REPORT_PERCENT_KEY,
      POLICY_REFRESHER_CHORE_REPORT_PERCENT_DEFAULT);
}