Java Code Examples for org.apache.hadoop.yarn.util.Clock

The following examples show how to use org.apache.hadoop.yarn.util.Clock. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: DefaultSpeculator.java    License: Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 2
Source Project: hadoop   Source File: InMemoryPlan.java    License: Apache License 2.0 6 votes vote down vote up
InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
    ReservationAgent agent, Resource totalCapacity, long step,
    ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
    String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) {
  this.queueMetrics = queueMetrics;
  this.policy = policy;
  this.agent = agent;
  this.step = step;
  this.totalCapacity = totalCapacity;
  this.resCalc = resCalc;
  this.minAlloc = minAlloc;
  this.maxAlloc = maxAlloc;
  this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
  this.queueName = queueName;
  this.replanner = replanner;
  this.getMoveOnExpiry = getMoveOnExpiry;
  this.clock = clock;
}
 
Example 3
Source Project: tez   Source File: MockDAGAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public MockDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId,
    String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime,
    boolean isSession, String workingDirectory, String[] localDirs, String[] logDirs,
    AtomicBoolean launcherGoFlag, boolean initFailFlag, boolean startFailFlag,
    Credentials credentials, String jobUserName, int handlerConcurrency, int numConcurrentContainers) {
  super(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, clock, appSubmitTime,
      isSession, workingDirectory, localDirs, logDirs,  new TezApiVersionInfo().getVersion(),
      credentials, jobUserName, null);
  shutdownHandler = new MockDAGAppMasterShutdownHandler();
  this.launcherGoFlag = launcherGoFlag;
  this.initFailFlag = initFailFlag;
  this.startFailFlag = startFailFlag;
  Preconditions.checkArgument(handlerConcurrency > 0);
  this.handlerConcurrency = handlerConcurrency;
  this.numConcurrentContainers = numConcurrentContainers;
}
 
Example 4
Source Project: hadoop   Source File: DefaultSpeculator.java    License: Apache License 2.0 6 votes vote down vote up
public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 
Example 5
Source Project: hadoop   Source File: MRAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 
Example 6
Source Project: incubator-tez   Source File: DAGAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  // TODO Metrics
  //this.metrics = DAGAppMetrics.create();
  LOG.info("Created DAGAppMaster for application " + applicationAttemptId);
}
 
Example 7
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 
Example 8
Source Project: big-c   Source File: TestReservationInputValidator.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
  clock = mock(Clock.class);
  plan = mock(Plan.class);
  rSystem = mock(ReservationSystem.class);
  plans.put(PLAN_NAME, plan);
  rrValidator = new ReservationInputValidator(clock);
  when(clock.getTime()).thenReturn(1L);
  ResourceCalculator rCalc = new DefaultResourceCalculator();
  Resource resource = Resource.newInstance(10240, 10);
  when(plan.getResourceCalculator()).thenReturn(rCalc);
  when(plan.getTotalCapacity()).thenReturn(resource);
  when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn(
      PLAN_NAME);
  when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan);
}
 
Example 9
Source Project: Bats   Source File: StreamingContainerManager.java    License: Apache License 2.0 5 votes vote down vote up
public StreamingContainerManager(LogicalPlan dag, boolean enableEventRecording, Clock clock)
{
  this.clock = clock;
  this.vars = new FinalVars(dag, clock.getTime());
  poolExecutor = Executors.newFixedThreadPool(4);
  // setup prior to plan creation for event recording
  if (enableEventRecording) {
    this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  }
  this.plan = new PhysicalPlan(dag, this);
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example 10
Source Project: hadoop   Source File: CapacitySchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
  super.init(clock, sched, plans);
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
  if (!(sched instanceof CapacityScheduler)) {
    throw new YarnRuntimeException(
        "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
  }
  this.cs = (CapacityScheduler) sched;
}
 
Example 11
Source Project: big-c   Source File: TaskHeartbeatHandler.java    License: Apache License 2.0 5 votes vote down vote up
public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock,
    int numThreads) {
  super("TaskHeartbeatHandler");
  this.eventHandler = eventHandler;
  this.clock = clock;
  runningAttempts =
    new ConcurrentHashMap<TaskAttemptId, ReportTime>(16, 0.75f, numThreads);
}
 
Example 12
Source Project: hadoop   Source File: FairSchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 
Example 13
Source Project: big-c   Source File: TestFairSchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 
Example 14
Source Project: big-c   Source File: TestHistoryFileManager.java    License: Apache License 2.0 5 votes vote down vote up
private void testCreateHistoryDirs(Configuration conf, Clock clock)
    throws Exception {
  conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID());
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID());
  HistoryFileManager hfm = new HistoryFileManager();
  hfm.conf = conf;
  hfm.createHistoryDirs(clock, 500, 2000);
}
 
Example 15
Source Project: tez   Source File: TaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskImpl(TezVertexID vertexId, int taskIndex,
    EventHandler eventHandler, Configuration conf,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
    Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
    boolean leafVertex, Resource resource,
    ContainerContext containerContext,
    StateChangeNotifier stateChangeNotifier,
    Vertex vertex) {
  this.conf = conf;
  this.clock = clock;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  readLock = readWriteLock.readLock();
  writeLock = readWriteLock.writeLock();
  this.attempts = Collections.emptyMap();
  maxFailedAttempts = vertex.getVertexConfig().getMaxFailedTaskAttempts();
  taskId = TezTaskID.getInstance(vertexId, taskIndex);
  this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
  this.taskHeartbeatHandler = thh;
  this.eventHandler = eventHandler;
  this.appContext = appContext;
  this.stateChangeNotifier = stateChangeNotifier;
  this.vertex = vertex;
  this.leafVertex = leafVertex;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskRecoveryData(taskId);
  stateMachine = new StateMachineTez<TaskStateInternal, TaskEventType, TaskEvent, TaskImpl>(
      stateMachineFactory.make(this), this);
  augmentStateMachine();
}
 
Example 16
Source Project: hadoop   Source File: TestCapacitySchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 
Example 17
Source Project: hadoop   Source File: TestFairSchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 
Example 18
Source Project: tez   Source File: TestVertexImpl2.java    License: Apache License 2.0 5 votes vote down vote up
VertexWrapper(AppContext appContext, VertexPlan vertexPlan, Configuration conf,
              boolean checkVertexOnlyConf) {
  if (appContext == null) {
    mockAppContext = createDefaultMockAppContext();
    DAG mockDag = mock(DAG.class);
    doReturn(new Credentials()).when(mockDag).getCredentials();
    doReturn(mockDag).when(mockAppContext).getCurrentDAG();
  } else {
    mockAppContext = appContext;
  }

  Configuration dagConf = new Configuration(false);
  dagConf.set("abc1", "xyz1");
  dagConf.set("foo1", "bar1");

  this.vertexPlan = vertexPlan;

  vertex =
      new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
          "testvertex", conf, mock(EventHandler.class), mock(TaskCommunicatorManagerInterface.class),
          mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
          VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
          new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class),
          dagConf);

  if (checkVertexOnlyConf) {
    Assert.assertEquals("xyz1", vertex.vertexOnlyConf.get("abc1"));
    Assert.assertEquals("bar2", vertex.vertexOnlyConf.get("foo1"));
    Assert.assertEquals("bar", vertex.vertexOnlyConf.get("foo"));
  }

}
 
Example 19
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example 20
Source Project: hadoop   Source File: ReduceTaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 
Example 21
Source Project: hadoop   Source File: MapTaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
public MapTaskAttemptImpl(TaskId taskId, int attempt, 
    EventHandler eventHandler, Path jobFile, 
    int partition, TaskSplitMetaInfo splitInfo, JobConf conf,
    TaskAttemptListener taskAttemptListener, 
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(taskId, attempt, eventHandler, 
      taskAttemptListener, jobFile, partition, conf, splitInfo.getLocations(),
      jobToken, credentials, clock, appContext);
  this.splitInfo = splitInfo;
}
 
Example 22
Source Project: hadoop   Source File: MapTaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
 
Example 23
Source Project: big-c   Source File: ReduceTaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 
Example 24
Source Project: tez   Source File: DAGAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession, String workingDirectory,
    String [] localDirs, String[] logDirs, String clientVersion,
    Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  this.workingDirectory = workingDirectory;
  this.localDirs = localDirs;
  this.logDirs = logDirs;
  this.shutdownHandler = createShutdownHandler();
  this.dagVersionInfo = new TezDagVersionInfo();
  this.clientVersion = clientVersion;
  this.amCredentials = credentials;
  this.amPluginDescriptorProto = pluginDescriptorProto;
  this.appMasterUgi = UserGroupInformation
      .createRemoteUser(jobUserName);
  this.appMasterUgi.addCredentials(amCredentials);

  this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
      this.containerID.toString(), this.appMasterUgi.getShortUserName());

  LOG.info("Created DAGAppMaster for application " + applicationAttemptId
      + ", versionInfo=" + dagVersionInfo.toString());
  TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
 
Example 25
Source Project: incubator-tez   Source File: AMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 
Example 26
Source Project: big-c   Source File: TestCapacitySchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 
Example 27
Source Project: hadoop   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
private TaskAttemptImpl createMapTaskAttemptImplForTest(
    EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  Path jobFile = mock(Path.class);
  JobConf jobConf = new JobConf();
  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          taskSplitMetaInfo, jobConf, taListener, null,
          null, clock, null);
  return taImpl;
}
 
Example 28
Source Project: hadoop   Source File: TestTaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
public MockTaskImpl(JobId jobId, int partition,
    EventHandler eventHandler, Path remoteJobConfFile, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock, int startCount,
    MRAppMetrics metrics, AppContext appContext, TaskType taskType) {
  super(jobId, taskType , partition, eventHandler,
      remoteJobConfFile, conf, taskAttemptListener,
      jobToken, credentials, clock,
      startCount, metrics, appContext);
  this.taskType = taskType;
}
 
Example 29
Source Project: tez   Source File: DatePartitionedLogger.java    License: Apache License 2.0 5 votes vote down vote up
public DatePartitionedLogger(Parser<T> parser, Path baseDir, Configuration conf, Clock clock)
    throws IOException {
  this.conf = new Configuration(conf);
  this.clock = clock;
  this.parser = parser;
  createDirIfNotExists(baseDir);
  this.basePath = baseDir.getFileSystem(conf).resolvePath(baseDir);
  FsPermission.setUMask(this.conf, FILE_UMASK);
}
 
Example 30
Source Project: tez   Source File: MockTezClient.java    License: Apache License 2.0 5 votes vote down vote up
MockTezClient(String name, TezConfiguration tezConf, boolean isSession,
    Map<String, LocalResource> localResources, Credentials credentials,
    Clock clock, AtomicBoolean mockAppLauncherGoFlag,
    boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) {
  super(name, tezConf, isSession, localResources, credentials);
  this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, 
      concurrency, containers);
}