org.apache.hadoop.yarn.util.Clock Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.Clock. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DefaultSpeculator.java    From big-c with Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example #2
Source File: InMemoryPlan.java    From hadoop with Apache License 2.0 6 votes vote down vote up
InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
    ReservationAgent agent, Resource totalCapacity, long step,
    ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
    String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) {
  this.queueMetrics = queueMetrics;
  this.policy = policy;
  this.agent = agent;
  this.step = step;
  this.totalCapacity = totalCapacity;
  this.resCalc = resCalc;
  this.minAlloc = minAlloc;
  this.maxAlloc = maxAlloc;
  this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
  this.queueName = queueName;
  this.replanner = replanner;
  this.getMoveOnExpiry = getMoveOnExpiry;
  this.clock = clock;
}
 
Example #3
Source File: MockDAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up
public MockDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId,
    String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime,
    boolean isSession, String workingDirectory, String[] localDirs, String[] logDirs,
    AtomicBoolean launcherGoFlag, boolean initFailFlag, boolean startFailFlag,
    Credentials credentials, String jobUserName, int handlerConcurrency, int numConcurrentContainers) {
  super(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, clock, appSubmitTime,
      isSession, workingDirectory, localDirs, logDirs,  new TezApiVersionInfo().getVersion(),
      credentials, jobUserName, null);
  shutdownHandler = new MockDAGAppMasterShutdownHandler();
  this.launcherGoFlag = launcherGoFlag;
  this.initFailFlag = initFailFlag;
  this.startFailFlag = startFailFlag;
  Preconditions.checkArgument(handlerConcurrency > 0);
  this.handlerConcurrency = handlerConcurrency;
  this.numConcurrentContainers = numConcurrentContainers;
}
 
Example #4
Source File: DefaultSpeculator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example #5
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 
Example #6
Source File: DAGAppMaster.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  // TODO Metrics
  //this.metrics = DAGAppMetrics.create();
  LOG.info("Created DAGAppMaster for application " + applicationAttemptId);
}
 
Example #7
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 
Example #8
Source File: TestReservationInputValidator.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
  clock = mock(Clock.class);
  plan = mock(Plan.class);
  rSystem = mock(ReservationSystem.class);
  plans.put(PLAN_NAME, plan);
  rrValidator = new ReservationInputValidator(clock);
  when(clock.getTime()).thenReturn(1L);
  ResourceCalculator rCalc = new DefaultResourceCalculator();
  Resource resource = Resource.newInstance(10240, 10);
  when(plan.getResourceCalculator()).thenReturn(rCalc);
  when(plan.getTotalCapacity()).thenReturn(resource);
  when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn(
      PLAN_NAME);
  when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan);
}
 
Example #9
Source File: StreamingContainerManager.java    From Bats with Apache License 2.0 5 votes vote down vote up
public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example #10
Source File: CapacitySchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
  super.init(clock, sched, plans);
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
  if (!(sched instanceof CapacityScheduler)) {
    throw new YarnRuntimeException(
        "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
  }
  this.cs = (CapacityScheduler) sched;
}
 
Example #11
Source File: TaskHeartbeatHandler.java    From big-c with Apache License 2.0 5 votes vote down vote up
public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock,
    int numThreads) {
  super("TaskHeartbeatHandler");
  this.eventHandler = eventHandler;
  this.clock = clock;
  runningAttempts =
    new ConcurrentHashMap<TaskAttemptId, ReportTime>(16, 0.75f, numThreads);
}
 
Example #12
Source File: FairSchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 
Example #13
Source File: TestFairSchedulerPlanFollower.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 
Example #14
Source File: TestHistoryFileManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void testCreateHistoryDirs(Configuration conf, Clock clock)
    throws Exception {
  conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID());
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID());
  HistoryFileManager hfm = new HistoryFileManager();
  hfm.conf = conf;
  hfm.createHistoryDirs(clock, 500, 2000);
}
 
Example #15
Source File: TaskImpl.java    From tez with Apache License 2.0 5 votes vote down vote up
public TaskImpl(TezVertexID vertexId, int taskIndex,
    EventHandler eventHandler, Configuration conf,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
    Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
    boolean leafVertex, Resource resource,
    ContainerContext containerContext,
    StateChangeNotifier stateChangeNotifier,
    Vertex vertex) {
  this.conf = conf;
  this.clock = clock;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  readLock = readWriteLock.readLock();
  writeLock = readWriteLock.writeLock();
  this.attempts = Collections.emptyMap();
  maxFailedAttempts = vertex.getVertexConfig().getMaxFailedTaskAttempts();
  taskId = TezTaskID.getInstance(vertexId, taskIndex);
  this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
  this.taskHeartbeatHandler = thh;
  this.eventHandler = eventHandler;
  this.appContext = appContext;
  this.stateChangeNotifier = stateChangeNotifier;
  this.vertex = vertex;
  this.leafVertex = leafVertex;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskRecoveryData(taskId);
  stateMachine = new StateMachineTez<TaskStateInternal, TaskEventType, TaskEvent, TaskImpl>(
      stateMachineFactory.make(this), this);
  augmentStateMachine();
}
 
Example #16
Source File: TestCapacitySchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 
Example #17
Source File: TestFairSchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 
Example #18
Source File: TestVertexImpl2.java    From tez with Apache License 2.0 5 votes vote down vote up
VertexWrapper(AppContext appContext, VertexPlan vertexPlan, Configuration conf,
              boolean checkVertexOnlyConf) {
  if (appContext == null) {
    mockAppContext = createDefaultMockAppContext();
    DAG mockDag = mock(DAG.class);
    doReturn(new Credentials()).when(mockDag).getCredentials();
    doReturn(mockDag).when(mockAppContext).getCurrentDAG();
  } else {
    mockAppContext = appContext;
  }

  Configuration dagConf = new Configuration(false);
  dagConf.set("abc1", "xyz1");
  dagConf.set("foo1", "bar1");

  this.vertexPlan = vertexPlan;

  vertex =
      new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
          "testvertex", conf, mock(EventHandler.class), mock(TaskCommunicatorManagerInterface.class),
          mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
          VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
          new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class),
          dagConf);

  if (checkVertexOnlyConf) {
    Assert.assertEquals("xyz1", vertex.vertexOnlyConf.get("abc1"));
    Assert.assertEquals("bar2", vertex.vertexOnlyConf.get("foo1"));
    Assert.assertEquals("bar", vertex.vertexOnlyConf.get("foo"));
  }

}
 
Example #19
Source File: TestProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example #20
Source File: ReduceTaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 
Example #21
Source File: MapTaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MapTaskAttemptImpl(TaskId taskId, int attempt, 
    EventHandler eventHandler, Path jobFile, 
    int partition, TaskSplitMetaInfo splitInfo, JobConf conf,
    TaskAttemptListener taskAttemptListener, 
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(taskId, attempt, eventHandler, 
      taskAttemptListener, jobFile, partition, conf, splitInfo.getLocations(),
      jobToken, credentials, clock, appContext);
  this.splitInfo = splitInfo;
}
 
Example #22
Source File: MapTaskImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
 
Example #23
Source File: ReduceTaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 
Example #24
Source File: DAGAppMaster.java    From tez with Apache License 2.0 5 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession, String workingDirectory,
    String [] localDirs, String[] logDirs, String clientVersion,
    Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  this.workingDirectory = workingDirectory;
  this.localDirs = localDirs;
  this.logDirs = logDirs;
  this.shutdownHandler = createShutdownHandler();
  this.dagVersionInfo = new TezDagVersionInfo();
  this.clientVersion = clientVersion;
  this.amCredentials = credentials;
  this.amPluginDescriptorProto = pluginDescriptorProto;
  this.appMasterUgi = UserGroupInformation
      .createRemoteUser(jobUserName);
  this.appMasterUgi.addCredentials(amCredentials);

  this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
      this.containerID.toString(), this.appMasterUgi.getShortUserName());

  LOG.info("Created DAGAppMaster for application " + applicationAttemptId
      + ", versionInfo=" + dagVersionInfo.toString());
  TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
 
Example #25
Source File: AMContainerImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 
Example #26
Source File: TestCapacitySchedulerPlanFollower.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 
Example #27
Source File: TestTaskAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private TaskAttemptImpl createMapTaskAttemptImplForTest(
    EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  Path jobFile = mock(Path.class);
  JobConf jobConf = new JobConf();
  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          taskSplitMetaInfo, jobConf, taListener, null,
          null, clock, null);
  return taImpl;
}
 
Example #28
Source File: TestTaskImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MockTaskImpl(JobId jobId, int partition,
    EventHandler eventHandler, Path remoteJobConfFile, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock, int startCount,
    MRAppMetrics metrics, AppContext appContext, TaskType taskType) {
  super(jobId, taskType , partition, eventHandler,
      remoteJobConfFile, conf, taskAttemptListener,
      jobToken, credentials, clock,
      startCount, metrics, appContext);
  this.taskType = taskType;
}
 
Example #29
Source File: DatePartitionedLogger.java    From tez with Apache License 2.0 5 votes vote down vote up
public DatePartitionedLogger(Parser<T> parser, Path baseDir, Configuration conf, Clock clock)
    throws IOException {
  this.conf = new Configuration(conf);
  this.clock = clock;
  this.parser = parser;
  createDirIfNotExists(baseDir);
  this.basePath = baseDir.getFileSystem(conf).resolvePath(baseDir);
  FsPermission.setUMask(this.conf, FILE_UMASK);
}
 
Example #30
Source File: MockTezClient.java    From tez with Apache License 2.0 5 votes vote down vote up
MockTezClient(String name, TezConfiguration tezConf, boolean isSession,
    Map<String, LocalResource> localResources, Credentials credentials,
    Clock clock, AtomicBoolean mockAppLauncherGoFlag,
    boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) {
  super(name, tezConf, isSession, localResources, credentials);
  this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, 
      concurrency, containers);
}