Java Code Examples for org.apache.hadoop.yarn.util.Clock#getTime()

The following examples show how to use org.apache.hadoop.yarn.util.Clock#getTime() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: MRAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 
Example 2
Source Project: hadoop   File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 
Example 3
Source Project: big-c   File: MRAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 
Example 4
Source Project: big-c   File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 
Example 5
Source Project: incubator-tez   File: DAGAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  // TODO Metrics
  //this.metrics = DAGAppMetrics.create();
  LOG.info("Created DAGAppMaster for application " + applicationAttemptId);
}
 
Example 6
Source Project: Bats   File: StreamingContainerManager.java    License: Apache License 2.0 5 votes vote down vote up
public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example 7
public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example 8
Source Project: incubator-tez   File: AMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 
Example 9
Source Project: tez   File: DAGAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession, String workingDirectory,
    String [] localDirs, String[] logDirs, String clientVersion,
    Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  this.workingDirectory = workingDirectory;
  this.localDirs = localDirs;
  this.logDirs = logDirs;
  this.shutdownHandler = createShutdownHandler();
  this.dagVersionInfo = new TezDagVersionInfo();
  this.clientVersion = clientVersion;
  this.amCredentials = credentials;
  this.amPluginDescriptorProto = pluginDescriptorProto;
  this.appMasterUgi = UserGroupInformation
      .createRemoteUser(jobUserName);
  this.appMasterUgi.addCredentials(amCredentials);

  this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
      this.containerID.toString(), this.appMasterUgi.getShortUserName());

  LOG.info("Created DAGAppMaster for application " + applicationAttemptId
      + ", versionInfo=" + dagVersionInfo.toString());
  TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
 
Example 10
Source Project: tez   File: AMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 
Example 11
Source Project: Bats   File: StreamingContainerManager.java    License: Apache License 2.0 4 votes vote down vote up
public UpdateCheckpointsContext(Clock clock, boolean recovery, Map<OperatorMeta, Set<OperatorMeta>> checkpointGroups)
{
  this.currentTms = clock.getTime();
  this.recovery = recovery;
  this.checkpointGroups = checkpointGroups;
}
 
Example 12
Source Project: XLearning   File: JobHistoryServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  FileSystem fs;
  Configuration conf = new XLearningConfiguration();
  Path historyLog = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR));
  Path eventLog = new Path(conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR));
  int monitorInterval = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL);
  int logMaxAge = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_MAX_AGE_MS,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_MAX_AGE_MS);
  final Clock clock = new SystemClock();
  while (!Thread.currentThread().isInterrupted()) {
    try {
      LOG.info("Start delete log thread.");
      Long currentClock = clock.getTime();
      fs = FileSystem.get(conf);
      FileStatus[] allHistoryLog = fs.listStatus(historyLog);
      LOG.info("historyLog:" + historyLog);
      for (FileStatus historyLogPer : allHistoryLog) {
        LOG.info(historyLogPer.getPath() + ":" + String.valueOf(currentClock - historyLogPer.getModificationTime()));
        if ((currentClock - historyLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(historyLogPer.getPath());
        }
      }
      FileStatus[] allEventLog = fs.listStatus(eventLog);
      LOG.info("eventLog:" + eventLog);
      for (FileStatus eventLogPer : allEventLog) {
        LOG.info(eventLogPer.getPath() + ":" + String.valueOf(currentClock - eventLogPer.getModificationTime()));
        if ((currentClock - eventLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(eventLogPer.getPath());
        }
      }

      Thread.sleep(monitorInterval);
    } catch (Exception e) {
      LOG.info("HistoryLog delete thread interrupted. " + e);
      break;
    }
  }
}
 
Example 13
public UpdateCheckpointsContext(Clock clock, boolean recovery, Map<OperatorMeta, Set<OperatorMeta>> checkpointGroups)
{
  this.currentTms = clock.getTime();
  this.recovery = recovery;
  this.checkpointGroups = checkpointGroups;
}
 
Example 14
Source Project: tez   File: TaskAttemptImpl.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
    TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
    boolean isRescheduled,
    Resource resource, ContainerContext containerContext, boolean leafVertex,
    Task task, TaskLocationHint locationHint, TaskSpec taskSpec,
    TezTaskAttemptID schedulingCausalTA) {

  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.attemptId = attemptId;
  this.eventHandler = eventHandler;
  //Reported status
  this.conf = conf;
  this.clock = clock;
  this.taskHeartbeatHandler = taskHeartbeatHandler;
  this.appContext = appContext;
  this.vertex = task.getVertex();
  this.task = task;
  this.locationHint = locationHint;
  this.taskSpec = taskSpec;
  this.creationCausalTA = schedulingCausalTA;
  this.creationTime = clock.getTime();

  this.reportedStatus = new TaskAttemptStatus(this.attemptId);
  initTaskAttemptStatus(reportedStatus);
  RackResolver.init(conf);
  this.stateMachine = stateMachineFactory.make(this);
  this.isRescheduled = isRescheduled;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.leafVertex = leafVertex;
  this.hungIntervalMax = conf.getLong(
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, 
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT);

  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskAttemptRecoveryData(attemptId);
}