Java Code Examples for org.apache.hadoop.metrics.MetricsUtil#createRecord()

ShuffleClientMetrics(String dagName, String vertexName, int taskIndex, Configuration conf, 
    String user) {
  this.numCopiers = 
      conf.getInt(
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT);

  MetricsContext metricsContext = MetricsUtil.getContext(Constants.TEZ);
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", user);
  this.shuffleMetrics.setTag("dagName", dagName);
  this.shuffleMetrics.setTag("taskId", TezRuntimeUtils.getTaskIdentifier(vertexName, taskIndex));
  this.shuffleMetrics.setTag("sessionId", 
      conf.get(
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID,
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID_DEFAULT));
  metricsContext.registerUpdater(this);
}
 

public HighTideNodeMetrics(Configuration conf, HighTideNode hightideNode) {
  String sessionId = conf.get("session.id");
  // Initiate Java VM metrics
  JvmMetrics.init("HighTideNode", sessionId);


  // Now the Mbean for the name node - this also registers the MBean
  hightidenodeActivityMBean = new HighTideNodeActivityMBean(registry);

  // Create a record for HighTideNode metrics
  MetricsContext metricsContext = MetricsUtil.getContext("dfs");
  metricsRecord = MetricsUtil.createRecord(metricsContext, "hightidenode");
  metricsRecord.setTag("sessionId", sessionId);
  metricsContext.registerUpdater(this);
  LOG.info("Initializing HighTideNodeMetrics using context object:" +
            metricsContext.getClass().getName());
}
 

public RpcMetrics(String hostName, String port, Server server) {
  myServer = server;
  context = MetricsUtil.getContext("rpc");
  metricsRecord = MetricsUtil.createRecord(context, "metrics");

  metricsRecord.setTag("port", port);

  LOG.info("Initializing RPC Metrics with hostName=" 
      + hostName + ", port=" + port);

  context.registerUpdater(this);
  
  // Need to clean up the interface to RpcMgt - don't need both metrics and server params
  rpcMBean = new RpcActivityMBean(registry, hostName, port);
}
 

/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 

public DataNodeMetrics(Configuration conf, String storageId) {
  String sessionId = conf.get("session.id"); 
  // Initiate reporting of Java VM metrics
  JvmMetrics.init("DataNode", sessionId);
  

  // Now the MBean for the data node
  datanodeActivityMBean = new DataNodeActivityMBean(registry, storageId);
  
  // Create record for DataNode metrics
  MetricsContext context = MetricsUtil.getContext("dfs");
  metricsRecord = MetricsUtil.createRecord(context, "datanode");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

ShuffleClientMetrics(JobConf conf) {
  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", conf.getUser());
  this.shuffleMetrics.setTag("jobName", conf.getJobName());
  this.shuffleMetrics.setTag("jobId", ReduceTask.this.getJobID().toString());
  this.shuffleMetrics.setTag("taskId", getTaskID().toString());
  this.shuffleMetrics.setTag("sessionId", conf.getSessionId());
  metricsContext.registerUpdater(this);
}
 

ShuffleServerMetrics(JobConf conf) {
  MetricsContext context = MetricsUtil.getContext("mapred");
  shuffleMetricsRecord =
                       MetricsUtil.createRecord(context, "shuffleOutput");
  this.shuffleMetricsRecord.setTag("sessionId", conf.getSessionId());
  context.registerUpdater(this);
}
 

public LocalJobRunnerMetrics(JobConf conf) {
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  // record name is jobtracker for compatibility 
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 

public JobTrackerMetricsInst(JobTracker tracker, JobConf conf) {
  super(tracker, conf);
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

public LookasideMetrics() {
  // Create a record for LookasideCache metrics
  MetricsContext metricsContext = MetricsUtil.getContext("lookasideCache");
  metricsRecord = MetricsUtil.createRecord(metricsContext,
                                           "LookasideFileSystem");
  metricsContext.registerUpdater(this);

}
 

public DFSClientMetrics() {
	// Create a record for FSNamesystem metrics
	MetricsContext metricsContext = MetricsUtil.getContext("hdfsclient");
	metricsRecord = MetricsUtil.createRecord(metricsContext, "DFSClient");
	metricsContext.registerUpdater(this);

}
 

public TaskTrackerMetricsInst(TaskTracker t) {
  super(t);
  JobConf conf = tt.getJobConf();
  String sessionId = conf.getSessionId();
  // Initiate Java VM Metrics
  JvmMetrics.init("TaskTracker", sessionId);
  // Create a record for Task Tracker metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "tasktracker"); //guaranteed never null
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 

/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 

ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf) {
  this.numCopiers = jobConf.getInt(MRJobConfig.SHUFFLE_PARALLEL_COPIES, 5);

  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", jobConf.getUser());
  this.shuffleMetrics.setTag("jobName", jobConf.getJobName());
  this.shuffleMetrics.setTag("jobId", reduceId.getJobID().toString());
  this.shuffleMetrics.setTag("taskId", reduceId.toString());
  this.shuffleMetrics.setTag("sessionId", jobConf.getSessionId());
  metricsContext.registerUpdater(this);
}
 

JobInProgress(JobID jobid, JobTracker jobtracker,
              JobConf default_conf, String user, int rCount)
throws IOException {
  this.restartCount = rCount;
  this.jobId = jobid;
  String url = "http://" + jobtracker.getJobTrackerMachine() + ":"
      + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
  this.jobtracker = jobtracker;
  this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP);
  this.jobtracker.getInstrumentation().addPrepJob(conf, jobid);
  this.startTime = JobTracker.getClock().getTime();
  status.setStartTime(startTime);
  this.localFs = FileSystem.getLocal(default_conf);

  JobConf default_job_conf = new JobConf(default_conf);
  this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR
                                                    +"/"+jobid + ".xml");

  if (user == null) {
    this.user = conf.getUser();
  } else {
    this.user = user;
  }
  LOG.info("User : " +  this.user);

  Path jobDir = jobtracker.getSystemDirectoryForJob(jobId);
  FileSystem fs = jobDir.getFileSystem(default_conf);
  jobFile = new Path(jobDir, "job.xml");

  if (!localFs.exists(localJobFile)) {
    fs.copyToLocalFile(jobFile, localJobFile);
  }

  conf = new JobConf(localJobFile);
  this.priority = conf.getJobPriority();
  this.status.setJobPriority(this.priority);
  this.profile = new JobProfile(user, jobid,
                                jobFile.toString(), url, conf.getJobName(),
                                conf.getQueueName());

  this.numMapTasks = conf.getNumMapTasks();
  this.numReduceTasks = conf.getNumReduceTasks();
  this.memoryPerMap = conf.getMemoryForMapTask();
  this.memoryPerReduce = conf.getMemoryForReduceTask();
  this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>
     (numMapTasks + numReduceTasks + 10);
  this.jobSetupCleanupNeeded = conf.getJobSetupCleanupNeeded();
  this.jobFinishWhenReducesDone = conf.getJobFinishWhenReducesDone();
  this.taskCleanupNeeded = conf.getTaskCleanupNeeded();
  LOG.info("Setup and cleanup tasks: jobSetupCleanupNeeded = " +
      jobSetupCleanupNeeded + ", taskCleanupNeeded = " + taskCleanupNeeded);

  this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
  this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
  this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();

  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
  this.jobMetrics.setTag("user", conf.getUser());
  this.jobMetrics.setTag("sessionId", conf.getSessionId());
  this.jobMetrics.setTag("jobName", conf.getJobName());
  this.jobMetrics.setTag("jobId", jobid.toString());
  hasSpeculativeMaps = conf.getMapSpeculativeExecution();
  hasSpeculativeReduces = conf.getReduceSpeculativeExecution();
  this.maxLevel = jobtracker.getNumTaskCacheLevels();
  this.anyCacheLevel = this.maxLevel+1;
  this.nonLocalMaps = new LinkedList<TaskInProgress>();
  this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
  this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
  this.nonRunningReduces = new LinkedList<TaskInProgress>();
  this.runningReduces = new LinkedHashSet<TaskInProgress>();
  this.resourceEstimator = new ResourceEstimator(this);
  this.slowTaskThreshold = Math.max(0.0f,
      conf.getFloat(SPECULATIVE_SLOWTASK_THRESHOLD,1.0f));
  this.speculativeCap = conf.getFloat(SPECULATIVECAP,0.1f);
  this.slowNodeThreshold = conf.getFloat(SPECULATIVE_SLOWNODE_THRESHOLD,1.0f);
  this.refreshTimeout = conf.getLong(JobInProgress.REFRESH_TIMEOUT,
      jobtracker.getJobTrackerReconfigurable().
      getInitialJobRefreshTimeoutMs());
  this.speculativeStddevMeanRatioMax = conf.getFloat(
      JobInProgress.SPECULATIVE_STDDEVMEANRATIO_MAX, 0.33f);
  this.speculativeMapUnfininshedThreshold = conf.getFloat(
      SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY,
      speculativeMapUnfininshedThreshold);
  this.speculativeReduceUnfininshedThreshold = conf.getFloat(
      SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY,
      speculativeReduceUnfininshedThreshold);
  enableNoFetchEmptyMapOutputs = conf.getBoolean(ENABLE_NO_FETCH_MAP_OUTPUTS, false);
  LOG.info(jobId + ": enableNoFetchEmptyMapOutputs = " + enableNoFetchEmptyMapOutputs);
}
 

public MultiTaskTrackerMetrics(List<TaskTracker> trackerList) {
  this.trackerList = trackerList;
  MetricsContext context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "multitasktracker");
  context.registerUpdater(this);
}
 

private void initialize(Configuration conf) {
  MetricsContext metricsContext = MetricsUtil.getContext("dfs");
  directoryMetrics = MetricsUtil.createRecord(metricsContext, "FSDirectory");
  directoryMetrics.setTag("sessionId", conf.get("session.id"));
}
 

@SuppressWarnings("deprecation")
  public SimulatorJobInProgress(JobID jobid, JobTracker jobtracker,
      JobConf default_conf, JobStory jobStory) {
    super(jobid, jobStory.getJobConf(), jobtracker);
    // jobSetupCleanupNeeded set to false in parent cstr, though
    // default is true

    restartCount = 0;
    jobSetupCleanupNeeded = false;
      
    this.memoryPerMap = conf.getMemoryForMapTask();
    this.memoryPerReduce = conf.getMemoryForReduceTask();
    this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();
    
    this.jobId = jobid;
    String url = "http://" + jobtracker.getJobTrackerMachine() + ":"
        + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
    this.jobtracker = jobtracker;
    this.conf = jobStory.getJobConf();
    this.priority = conf.getJobPriority();
    Path jobDir = jobtracker.getSystemDirectoryForJob(jobid);
    this.jobFile = new Path(jobDir, "job.xml");
    this.status = new JobStatus(jobid, 0.0f, 0.0f, 0.0f, 0.0f, JobStatus.PREP,
            priority, conf.getUser());
    this.profile = new JobProfile(jobStory.getUser(), jobid, this.jobFile
        .toString(), url, jobStory.getName(), conf.getQueueName());
    this.startTime = JobTracker.getClock().getTime();
    status.setStartTime(startTime);
    this.resourceEstimator = new ResourceEstimator(this);

    this.numMapTasks = jobStory.getNumberMaps();
    this.numReduceTasks = jobStory.getNumberReduces();
    this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>(numMapTasks
        + numReduceTasks + 10);

    this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
    this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
    MetricsContext metricsContext = MetricsUtil.getContext("mapred");
    this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
    this.jobMetrics.setTag("user", conf.getUser());
    this.jobMetrics.setTag("sessionId", conf.getSessionId());
    this.jobMetrics.setTag("jobName", conf.getJobName());
    this.jobMetrics.setTag("jobId", jobid.toString());

    this.maxLevel = jobtracker.getNumTaskCacheLevels();
    this.anyCacheLevel = this.maxLevel + 1;
    this.nonLocalMaps = new LinkedList<TaskInProgress>();
    this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
    this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
    this.nonRunningReduces = new LinkedList<TaskInProgress>();
    this.runningReduces = new LinkedHashSet<TaskInProgress>();
    this.slowTaskThreshold = Math.max(0.0f, conf.getFloat(
        "mapred.speculative.execution.slowTaskThreshold", 1.0f));
    this.speculativeCap = conf.getFloat(
        "mapred.speculative.execution.speculativeCap", 0.1f);
    this.slowNodeThreshold = conf.getFloat(
        "mapred.speculative.execution.slowNodeThreshold", 1.0f);

    this.jobStory = jobStory;
//    this.jobHistory = this.jobtracker.getJobHistory();
  }
 

public ProxyJobTracker(CoronaConf conf) throws IOException {
  this.conf = conf;
  fs = FileSystem.get(conf);
  String infoAddr =
    conf.get("mapred.job.tracker.corona.proxyaddr", "0.0.0.0:0");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String infoBindAddress = infoSocAddr.getHostName();
  int port = infoSocAddr.getPort();
  LOCALMACHINE = infoBindAddress;
  startTime = getClock().getTime();

  CoronaConf coronaConf = new CoronaConf(conf);
  InetSocketAddress rpcSockAddr = NetUtils.createSocketAddr(
    coronaConf.getProxyJobTrackerAddress());
  rpcServer = RPC.getServer(
    this,
    rpcSockAddr.getHostName(),
    rpcSockAddr.getPort(),
    conf.getInt("corona.proxy.job.tracker.handler.count", 10),
    false,
    conf);
  rpcServer.start();

  LOG.info("ProxyJobTracker RPC Server up at " +
    rpcServer.getListenerAddress());

  infoServer = new HttpServer("proxyjt", infoBindAddress, port,
                              port == 0, conf);
  infoServer.setAttribute("proxy.job.tracker", this);
  infoServer.setAttribute("conf", conf);
  infoServer.addServlet("proxy", "/proxy",
                        ProxyJobTrackerServlet.class);
  // initialize history parameters.
  JobConf jobConf = new JobConf(conf);
  boolean historyInitialized = JobHistory.init(
    this, jobConf, this.LOCALMACHINE, this.startTime);
  if (historyInitialized) {
    JobHistory.initDone(jobConf, fs);
    String historyLogDir =
        JobHistory.getCompletedJobHistoryLocation().toString();
    FileSystem historyFS = new Path(historyLogDir).getFileSystem(conf);
    infoServer.setAttribute("historyLogDir", historyLogDir);
    infoServer.setAttribute("fileSys", historyFS);
  }
  infoServer.start();
  LOCALPORT = infoServer.getPort();

  context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "proxyjobtracker");
  context.registerUpdater(this);

  expireUnusedFilesInCache = new ExpireUnusedFilesInCache(
    conf, getClock(), new Path(getSystemDir()), fs);

  sessionHistoryManager = new SessionHistoryManager();
  sessionHistoryManager.setConf(conf);

  try {
    String target = conf.getProxyJobTrackerThriftAddress();
    InetSocketAddress addr = NetUtils.createSocketAddr(target);
    LOG.info("Trying to start the Thrift Server at: " + target);
    ServerSocket serverSocket = new ServerSocket(addr.getPort());
    server = new TServerThread(
      TFactoryBasedThreadPoolServer.createNewServer(
        new CoronaProxyJobTrackerService.Processor(this),
        serverSocket,
        5000));
    server.start();
    LOG.info("Thrift server started on: " + target);
  } catch (IOException e) {
    LOG.info("Exception while starting the Thrift Server on CPJT: ", e);
  }
}