Java Code Examples for org.apache.hadoop.conf.Configuration#getInts()

The following examples show how to use org.apache.hadoop.conf.Configuration#getInts() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IPCLoggerChannelMetrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
  this.ch = ch;
  
  Configuration conf = new HdfsConfiguration();
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  if (intervals != null) {
    writeEndToEndLatencyQuantiles = new MutableQuantiles[intervals.length];
    writeRpcLatencyQuantiles = new MutableQuantiles[intervals.length];
    for (int i = 0; i < writeEndToEndLatencyQuantiles.length; i++) {
      int interval = intervals[i];
      writeEndToEndLatencyQuantiles[i] = registry.newQuantiles(
          "writesE2E" + interval + "s",
          "End-to-end time for write operations", "ops", "LatencyMicros", interval);
      writeRpcLatencyQuantiles[i] = registry.newQuantiles(
          "writesRpc" + interval + "s",
          "RPC RTT for write operations", "ops", "LatencyMicros", interval);
    }
  } else {
    writeEndToEndLatencyQuantiles = null;
    writeRpcLatencyQuantiles = null;
  }
}
Example 2
Source File: WeightedRoundRobinMultiplexer.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public WeightedRoundRobinMultiplexer(int aNumQueues, String ns,
  Configuration conf) {
  if (aNumQueues <= 0) {
    throw new IllegalArgumentException("Requested queues (" + aNumQueues +
      ") must be greater than zero.");
  }

  this.numQueues = aNumQueues;
  this.queueWeights = conf.getInts(ns + "." +
    IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY);

  if (this.queueWeights.length == 0) {
    this.queueWeights = getDefaultQueueWeights(this.numQueues);
  } else if (this.queueWeights.length != this.numQueues) {
    throw new IllegalArgumentException(ns + "." +
      IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY + " must specify exactly " +
      this.numQueues + " weights: one for each priority level.");
  }

  this.currentQueueIndex = new AtomicInteger(0);
  this.requestsLeft = new AtomicInteger(this.queueWeights[0]);

  LOG.info("WeightedRoundRobinMultiplexer is being used.");
}
Example 3
Source File: DecayRpcScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private static double[] parseThresholds(String ns, Configuration conf,
    int numQueues) {
  int[] percentages = conf.getInts(ns + "." +
    IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY);

  if (percentages.length == 0) {
    return getDefaultThresholds(numQueues);
  } else if (percentages.length != numQueues-1) {
    throw new IllegalArgumentException("Number of thresholds should be " +
      (numQueues-1) + ". Was: " + percentages.length);
  }

  // Convert integer percentages to decimals
  double[] decimals = new double[percentages.length];
  for (int i = 0; i < percentages.length; i++) {
    decimals[i] = percentages[i] / 100.0;
  }

  return decimals;
}
Example 4
Source File: IPCLoggerChannelMetrics.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private IPCLoggerChannelMetrics(IPCLoggerChannel ch) {
  this.ch = ch;
  
  Configuration conf = new HdfsConfiguration();
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  if (intervals != null) {
    writeEndToEndLatencyQuantiles = new MutableQuantiles[intervals.length];
    writeRpcLatencyQuantiles = new MutableQuantiles[intervals.length];
    for (int i = 0; i < writeEndToEndLatencyQuantiles.length; i++) {
      int interval = intervals[i];
      writeEndToEndLatencyQuantiles[i] = registry.newQuantiles(
          "writesE2E" + interval + "s",
          "End-to-end time for write operations", "ops", "LatencyMicros", interval);
      writeRpcLatencyQuantiles[i] = registry.newQuantiles(
          "writesRpc" + interval + "s",
          "RPC RTT for write operations", "ops", "LatencyMicros", interval);
    }
  } else {
    writeEndToEndLatencyQuantiles = null;
    writeRpcLatencyQuantiles = null;
  }
}
Example 5
Source File: WeightedRoundRobinMultiplexer.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public WeightedRoundRobinMultiplexer(int aNumQueues, String ns,
  Configuration conf) {
  if (aNumQueues <= 0) {
    throw new IllegalArgumentException("Requested queues (" + aNumQueues +
      ") must be greater than zero.");
  }

  this.numQueues = aNumQueues;
  this.queueWeights = conf.getInts(ns + "." +
    IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY);

  if (this.queueWeights.length == 0) {
    this.queueWeights = getDefaultQueueWeights(this.numQueues);
  } else if (this.queueWeights.length != this.numQueues) {
    throw new IllegalArgumentException(ns + "." +
      IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY + " must specify exactly " +
      this.numQueues + " weights: one for each priority level.");
  }

  this.currentQueueIndex = new AtomicInteger(0);
  this.requestsLeft = new AtomicInteger(this.queueWeights[0]);

  LOG.info("WeightedRoundRobinMultiplexer is being used.");
}
Example 6
Source File: DecayRpcScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private static double[] parseThresholds(String ns, Configuration conf,
    int numQueues) {
  int[] percentages = conf.getInts(ns + "." +
    IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY);

  if (percentages.length == 0) {
    return getDefaultThresholds(numQueues);
  } else if (percentages.length != numQueues-1) {
    throw new IllegalArgumentException("Number of thresholds should be " +
      (numQueues-1) + ". Was: " + percentages.length);
  }

  // Convert integer percentages to decimals
  double[] decimals = new double[percentages.length];
  for (int i = 0; i < percentages.length; i++) {
    decimals[i] = percentages[i] / 100.0;
  }

  return decimals;
}
Example 7
Source File: Nfs3Metrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static Nfs3Metrics create(Configuration conf, String gatewayName) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create(gatewayName, sessionId, ms);

  // Percentile measurement is [50th,75th,90th,95th,99th] currently 
  int[] intervals = conf
      .getInts(NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_KEY);
  return ms.register(new Nfs3Metrics(gatewayName, sessionId, intervals, jm));
}
Example 8
Source File: NameNodeMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  String processName = r.toString();
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
  
  // Percentile measurement is off by default, by watching no intervals
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  return ms.register(new NameNodeMetrics(processName, sessionId,
      intervals, jm));
}
Example 9
Source File: DataNodeMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static DataNodeMetrics create(Configuration conf, String dnName) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
  String name = "DataNodeActivity-"+ (dnName.isEmpty()
      ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt() 
          : dnName.replace(':', '-'));

  // Percentile measurement is off by default, by watching no intervals
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  
  return ms.register(name, null, new DataNodeMetrics(name, sessionId,
      intervals, jm));
}
Example 10
Source File: RpcMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}
Example 11
Source File: Nfs3Metrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static Nfs3Metrics create(Configuration conf, String gatewayName) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create(gatewayName, sessionId, ms);

  // Percentile measurement is [50th,75th,90th,95th,99th] currently 
  int[] intervals = conf
      .getInts(NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_KEY);
  return ms.register(new Nfs3Metrics(gatewayName, sessionId, intervals, jm));
}
Example 12
Source File: NameNodeMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  String processName = r.toString();
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
  
  // Percentile measurement is off by default, by watching no intervals
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  return ms.register(new NameNodeMetrics(processName, sessionId,
      intervals, jm));
}
Example 13
Source File: DataNodeMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static DataNodeMetrics create(Configuration conf, String dnName) {
  String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
  MetricsSystem ms = DefaultMetricsSystem.instance();
  JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
  String name = "DataNodeActivity-"+ (dnName.isEmpty()
      ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt() 
          : dnName.replace(':', '-'));

  // Percentile measurement is off by default, by watching no intervals
  int[] intervals = 
      conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
  
  return ms.register(name, null, new DataNodeMetrics(name, sessionId,
      intervals, jm));
}
Example 14
Source File: RpcMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up 
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}