Java Code Examples for org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree#getResourceCalculatorProcessTree()

The following examples show how to use org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree#getResourceCalculatorProcessTree() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ContainersMonitorImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private boolean isEnabled() {
  if (resourceCalculatorPlugin == null) {
          LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
      LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
    LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
        "needed. Not running the monitor-thread");
    return false;
  }

  return true;
}
Example 2
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private boolean isEnabled() {
  if (resourceCalculatorPlugin == null) {
          LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
      LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
    LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
        "needed. Not running the monitor-thread");
    return false;
  }

  return true;
}
Example 3
Source File: DAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up 
private void initResourceCalculatorPlugins() {
  Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class,
      ResourceCalculatorProcessTree.class);

  // this is set by YARN NM
  String pid = System.getenv().get("JVM_PID");
  // for the local debug test cases fallback to JVM hooks that are not portable
  if (pid == null || pid.length() == 0) {
    String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
    pid = processName.split("@")[0];
  }
  cpuPlugin = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, amConf);

  gcPlugin = new GcTimeUpdater(null);
}
Example 4
Source File: TaskCounterUpdater.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
private void initResourceCalculatorPlugin() {
  Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass(
      TezJobConfig.TEZ_RUNTIME_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
      ResourceCalculatorProcessTree.class); 

  pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
      System.getenv().get("JVM_PID"), clazz, conf);

  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
}
Example 5
Source File: TestTezMxBeanResourceCalculator.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws Exception {
  Configuration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class.getName());

  Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
      ResourceCalculatorProcessTree.class);
  resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
      "", clazz, conf);
}
Example 6
Source File: TaskCounterUpdater.java    From tez with Apache License 2.0 5 votes vote down vote up 
private void initResourceCalculatorPlugin() {
  Class<? extends ResourceCalculatorProcessTree> clazz = this.conf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class,
      ResourceCalculatorProcessTree.class); 

  pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, conf);

  LOG.info("Using ResourceCalculatorProcessTree : " + clazz.getName());
}
Example 7
Source File: Task.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void initialize(JobConf job, JobID id, 
                       Reporter reporter,
                       boolean useNewApi) throws IOException, 
                                                 ClassNotFoundException,
                                                 InterruptedException {
  jobContext = new JobContextImpl(job, id, reporter);
  taskContext = new TaskAttemptContextImpl(job, taskId, reporter);
  if (getState() == TaskStatus.State.UNASSIGNED) {
    setState(TaskStatus.State.RUNNING);
  }
  if (useNewApi) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("using new api for output committer");
    }
    outputFormat =
      ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job);
    committer = outputFormat.getOutputCommitter(taskContext);
  } else {
    committer = conf.getOutputCommitter();
  }
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    if ((committer instanceof FileOutputCommitter)) {
      FileOutputFormat.setWorkOutputPath(conf, 
        ((FileOutputCommitter)committer).getTaskAttemptPath(taskContext));
    } else {
      FileOutputFormat.setWorkOutputPath(conf, outputPath);
    }
  }
  committer.setupTask(taskContext);
  Class<? extends ResourceCalculatorProcessTree> clazz =
      conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE,
          null, ResourceCalculatorProcessTree.class);
  pTree = ResourceCalculatorProcessTree
          .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf);
  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
  if (pTree != null) {
    pTree.updateProcessTree();
    initCpuCumulativeTime = pTree.getCumulativeCpuTime();
  }
}
Example 8
Source File: Task.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void initialize(JobConf job, JobID id, 
                       Reporter reporter,
                       boolean useNewApi) throws IOException, 
                                                 ClassNotFoundException,
                                                 InterruptedException {
  jobContext = new JobContextImpl(job, id, reporter);
  taskContext = new TaskAttemptContextImpl(job, taskId, reporter);
  if (getState() == TaskStatus.State.UNASSIGNED) {
    setState(TaskStatus.State.RUNNING);
  }
  if (useNewApi) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("using new api for output committer");
    }
    outputFormat =
      ReflectionUtils.newInstance(taskContext.getOutputFormatClass(), job);
    committer = outputFormat.getOutputCommitter(taskContext);
  } else {
    committer = conf.getOutputCommitter();
  }
  Path outputPath = FileOutputFormat.getOutputPath(conf);
  if (outputPath != null) {
    if ((committer instanceof FileOutputCommitter)) {
      FileOutputFormat.setWorkOutputPath(conf, 
        ((FileOutputCommitter)committer).getTaskAttemptPath(taskContext));
    } else {
      FileOutputFormat.setWorkOutputPath(conf, outputPath);
    }
  }
  committer.setupTask(taskContext);
  Class<? extends ResourceCalculatorProcessTree> clazz =
      conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE,
          null, ResourceCalculatorProcessTree.class);
  pTree = ResourceCalculatorProcessTree
          .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf);
  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
  if (pTree != null) {
    pTree.updateProcessTree();
    initCpuCumulativeTime = pTree.getCumulativeCpuTime();
  }
}