Java Code Examples for org.apache.hadoop.yarn.util.Clock#getTime()

public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 

@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 

public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 

@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 

public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  // TODO Metrics
  //this.metrics = DAGAppMetrics.create();
  LOG.info("Created DAGAppMaster for application " + applicationAttemptId);
}
 

public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 

public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 

private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 

public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession, String workingDirectory,
    String [] localDirs, String[] logDirs, String clientVersion,
    Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  this.workingDirectory = workingDirectory;
  this.localDirs = localDirs;
  this.logDirs = logDirs;
  this.shutdownHandler = createShutdownHandler();
  this.dagVersionInfo = new TezDagVersionInfo();
  this.clientVersion = clientVersion;
  this.amCredentials = credentials;
  this.amPluginDescriptorProto = pluginDescriptorProto;
  this.appMasterUgi = UserGroupInformation
      .createRemoteUser(jobUserName);
  this.appMasterUgi.addCredentials(amCredentials);

  this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
      this.containerID.toString(), this.appMasterUgi.getShortUserName());

  LOG.info("Created DAGAppMaster for application " + applicationAttemptId
      + ", versionInfo=" + dagVersionInfo.toString());
  TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
 

private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 

public UpdateCheckpointsContext(Clock clock, boolean recovery, Map<OperatorMeta, Set<OperatorMeta>> checkpointGroups)
{
  this.currentTms = clock.getTime();
  this.recovery = recovery;
  this.checkpointGroups = checkpointGroups;
}
 

@Override
public void run() {
  FileSystem fs;
  Configuration conf = new XLearningConfiguration();
  Path historyLog = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR));
  Path eventLog = new Path(conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR));
  int monitorInterval = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL);
  int logMaxAge = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_MAX_AGE_MS,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_MAX_AGE_MS);
  final Clock clock = new SystemClock();
  while (!Thread.currentThread().isInterrupted()) {
    try {
      LOG.info("Start delete log thread.");
      Long currentClock = clock.getTime();
      fs = FileSystem.get(conf);
      FileStatus[] allHistoryLog = fs.listStatus(historyLog);
      LOG.info("historyLog:" + historyLog);
      for (FileStatus historyLogPer : allHistoryLog) {
        LOG.info(historyLogPer.getPath() + ":" + String.valueOf(currentClock - historyLogPer.getModificationTime()));
        if ((currentClock - historyLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(historyLogPer.getPath());
        }
      }
      FileStatus[] allEventLog = fs.listStatus(eventLog);
      LOG.info("eventLog:" + eventLog);
      for (FileStatus eventLogPer : allEventLog) {
        LOG.info(eventLogPer.getPath() + ":" + String.valueOf(currentClock - eventLogPer.getModificationTime()));
        if ((currentClock - eventLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(eventLogPer.getPath());
        }
      }

      Thread.sleep(monitorInterval);
    } catch (Exception e) {
      LOG.info("HistoryLog delete thread interrupted. " + e);
      break;
    }
  }
}
 

public UpdateCheckpointsContext(Clock clock, boolean recovery, Map<OperatorMeta, Set<OperatorMeta>> checkpointGroups)
{
  this.currentTms = clock.getTime();
  this.recovery = recovery;
  this.checkpointGroups = checkpointGroups;
}
 

@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
    TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
    boolean isRescheduled,
    Resource resource, ContainerContext containerContext, boolean leafVertex,
    Task task, TaskLocationHint locationHint, TaskSpec taskSpec,
    TezTaskAttemptID schedulingCausalTA) {

  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.attemptId = attemptId;
  this.eventHandler = eventHandler;
  //Reported status
  this.conf = conf;
  this.clock = clock;
  this.taskHeartbeatHandler = taskHeartbeatHandler;
  this.appContext = appContext;
  this.vertex = task.getVertex();
  this.task = task;
  this.locationHint = locationHint;
  this.taskSpec = taskSpec;
  this.creationCausalTA = schedulingCausalTA;
  this.creationTime = clock.getTime();

  this.reportedStatus = new TaskAttemptStatus(this.attemptId);
  initTaskAttemptStatus(reportedStatus);
  RackResolver.init(conf);
  this.stateMachine = stateMachineFactory.make(this);
  this.isRescheduled = isRescheduled;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.leafVertex = leafVertex;
  this.hungIntervalMax = conf.getLong(
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, 
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT);

  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskAttemptRecoveryData(attemptId);
}