Java Code Examples for org.apache.hadoop.yarn.util.SystemClock

The following examples show how to use org.apache.hadoop.yarn.util.SystemClock. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestQueueManager.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new FairSchedulerConfiguration();
  FairScheduler scheduler = mock(FairScheduler.class);
  AllocationConfiguration allocConf = new AllocationConfiguration(conf);
  when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
  when(scheduler.getConf()).thenReturn(conf);
  SystemClock clock = new SystemClock();
  when(scheduler.getClock()).thenReturn(clock);
  notEmptyQueues = new HashSet<FSQueue>();
  queueManager = new QueueManager(scheduler) {
    @Override
    public boolean isEmpty(FSQueue queue) {
      return !notEmptyQueues.contains(queue);
    }
  };
  FSQueueMetrics.forQueue("root", null, true, conf);
  queueManager.initialize(conf);
}
 
Example 2
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
    String user, int numSplits, AppContext appContext) {
  super(jobId, applicationAttemptId, conf, eventHandler,
      null, new JobTokenSecretManager(), new Credentials(),
      new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
      MRAppMetrics.create(), null, newApiCommitter, user,
      System.currentTimeMillis(), null, appContext, null, null);

  initTransition = getInitTransition(numSplits);
  localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
        EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
        JobEventType.JOB_INIT,
        // This is abusive.
        initTransition);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example 3
Source Project: big-c   Source File: TestQueueManager.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new FairSchedulerConfiguration();
  FairScheduler scheduler = mock(FairScheduler.class);
  AllocationConfiguration allocConf = new AllocationConfiguration(conf);
  when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
  when(scheduler.getConf()).thenReturn(conf);
  SystemClock clock = new SystemClock();
  when(scheduler.getClock()).thenReturn(clock);
  notEmptyQueues = new HashSet<FSQueue>();
  queueManager = new QueueManager(scheduler) {
    @Override
    public boolean isEmpty(FSQueue queue) {
      return !notEmptyQueues.contains(queue);
    }
  };
  FSQueueMetrics.forQueue("root", null, true, conf);
  queueManager.initialize(conf);
}
 
Example 4
Source Project: tez   Source File: LocalClient.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected DAGAppMaster createDAGAppMaster(ApplicationAttemptId applicationAttemptId,
                                          ContainerId cId, String currentHost, int nmPort,
                                          int nmHttpPort,
                                          Clock clock, long appSubmitTime, boolean isSession,
                                          String userDir,
                                          String[] localDirs, String[] logDirs,
                                          Credentials credentials, String jobUserName) throws
    IOException {

  // Read in additional information about external services
  AMPluginDescriptorProto amPluginDescriptorProto =
      TezUtilsInternal.readUserSpecifiedTezConfiguration(userDir)
          .getAmPluginDescriptor();

  return new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort,
      new SystemClock(), appSubmitTime, isSession, userDir, localDirs, logDirs,
      versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto);
}
 
Example 5
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static CommitterEventHandler createCommitterEventHandler(
    Dispatcher dispatcher, OutputCommitter committer) {
  final SystemClock clock = new SystemClock();
  AppContext appContext = mock(AppContext.class);
  when(appContext.getEventHandler()).thenReturn(
      dispatcher.getEventHandler());
  when(appContext.getClock()).thenReturn(clock);
  RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {
    @Override
    public long getLastHeartbeatTime() {
      return clock.getTime();
    }
    @Override
    public void runOnNextHeartbeat(Runnable callback) {
      callback.run();
    }
  };
  ApplicationAttemptId id = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
  when(appContext.getApplicationAttemptId()).thenReturn(id);
  CommitterEventHandler handler =
      new CommitterEventHandler(appContext, committer, heartbeatHandler);
  dispatcher.register(CommitterEventType.class, handler);
  return handler;
}
 
Example 6
Source Project: Bats   Source File: StreamingContainerManager.java    License: Apache License 2.0 5 votes vote down vote up
private StreamingContainerManager(CheckpointState checkpointedState, boolean enableEventRecording)
{
  this.vars = checkpointedState.finals;
  this.clock = new SystemClock();
  poolExecutor = Executors.newFixedThreadPool(4);
  this.plan = checkpointedState.physicalPlan;
  this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example 7
Source Project: attic-apex-core   Source File: StreamingContainerManager.java    License: Apache License 2.0 5 votes vote down vote up
private StreamingContainerManager(CheckpointState checkpointedState, boolean enableEventRecording)
{
  this.vars = checkpointedState.finals;
  this.clock = new SystemClock();
  poolExecutor = Executors.newFixedThreadPool(4);
  this.plan = checkpointedState.physicalPlan;
  this.eventBus = new MBassador<>(BusConfiguration.Default(1, 1, 1));
  this.journal = new Journal(this);
  init(enableEventRecording);
}
 
Example 8
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "MoveIntermediateToDone Thread #%d").build();
  moveToDoneExecutor = new ThreadPoolExecutor(numMoveThreads, numMoveThreads,
      1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);

  super.serviceInit(conf);
}
 
Example 9
Source Project: tez   Source File: TestTaskCommunicatorManager2.java    License: Apache License 2.0 5 votes vote down vote up
TaskCommunicatorManagerWrapperForTest() throws IOException, TezException {
  dagId = TezDAGID.getInstance(appId, 1);
  vertexId = TezVertexID.getInstance(dagId, 100);
  doReturn(eventHandler).when(appContext).getEventHandler();
  doReturn(dag).when(appContext).getCurrentDAG();
  doReturn(vertex).when(dag).getVertex(eq(vertexId));
  doReturn(new TaskAttemptEventInfo(0, new LinkedList<TezEvent>(), 0)).when(vertex)
      .getTaskAttemptTezEvents(any(TezTaskAttemptID.class), anyInt(), anyInt(), anyInt());
  doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
  doReturn(credentials).when(appContext).getAppCredentials();
  doReturn(appAcls).when(appContext).getApplicationACLs();
  doReturn(amContainerMap).when(appContext).getAllContainers();
  doReturn(new SystemClock()).when(appContext).getClock();

  NodeId nodeId = NodeId.newInstance("localhost", 0);
  AMContainer amContainer = mock(AMContainer.class);
  Container container = mock(Container.class);
  doReturn(nodeId).when(container).getNodeId();
  doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
  doReturn(container).when(amContainer).getContainer();

  userPayload = TezUtils.createUserPayloadFromConf(conf);

  taskCommunicatorManager =
      new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class),
          mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(
          TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
}
 
Example 10
Source Project: hadoop   Source File: TestRecovery.java    License: Apache License 2.0 5 votes vote down vote up
private MapTaskImpl getMockMapTask(long clusterTimestamp, EventHandler eh) {

    ApplicationId appId = ApplicationId.newInstance(clusterTimestamp, 1);
    JobId jobId = MRBuilderUtils.newJobId(appId, 1);

    int partitions = 2;

    Path remoteJobConfFile = mock(Path.class);
    JobConf conf = new JobConf();
    TaskAttemptListener taskAttemptListener = mock(TaskAttemptListener.class);
    Token<JobTokenIdentifier> jobToken =
        (Token<JobTokenIdentifier>) mock(Token.class);
    Credentials credentials = null;
    Clock clock = new SystemClock();
    int appAttemptId = 3;
    MRAppMetrics metrics = mock(MRAppMetrics.class);
    Resource minContainerRequirements = mock(Resource.class);
    when(minContainerRequirements.getMemory()).thenReturn(1000);

    ClusterInfo clusterInfo = mock(ClusterInfo.class);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getClusterInfo()).thenReturn(clusterInfo);

    TaskSplitMetaInfo taskSplitMetaInfo = mock(TaskSplitMetaInfo.class);
    MapTaskImpl mapTask = new MapTaskImpl(jobId, partitions,
        eh, remoteJobConfFile, conf,
        taskSplitMetaInfo, taskAttemptListener, jobToken, credentials, clock,
        appAttemptId, metrics, appContext);
    return mapTask;
  }
 
Example 11
Source Project: incubator-tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testLocalityRequest() {

  TaskAttemptImpl.ScheduleTaskattemptTransition sta =
      new TaskAttemptImpl.ScheduleTaskattemptTransition();

  EventHandler eventHandler = mock(EventHandler.class);
  Set<String> hosts = new TreeSet<String>();
  hosts.add("host1");
  hosts.add("host2");
  hosts.add("host3");
  TaskLocationHint locationHint = new TaskLocationHint(hosts, null);

  TezTaskID taskID = TezTaskID.getInstance(
      TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
      mock(TaskHeartbeatHandler.class), mock(AppContext.class),
      locationHint, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);

  TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class);

  sta.transition(taImpl, sEvent);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(1)).handle(arg.capture());
  if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
    fail("Second event not of type "
        + AMSchedulerEventTALaunchRequest.class.getName());
  }
  // TODO Move the Rack request check to the client after TEZ-125 is fixed.
  Set<String> requestedRacks = taImpl.taskRacks;
  assertEquals(1, requestedRacks.size());
  assertEquals(3, taImpl.taskHosts.size());
  for (int i = 0; i < 3; i++) {
    String host = ("host" + (i + 1));
    assertEquals(host, true, taImpl.taskHosts.contains(host));
  }
}
 
Example 12
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReportDiagnostics() throws Exception {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  final String diagMsg = "some diagnostic message";
  final JobDiagnosticsUpdateEvent diagUpdateEvent =
      new JobDiagnosticsUpdateEvent(jobId, diagMsg);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  JobImpl job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(diagUpdateEvent);
  String diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));

  job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  job.handle(diagUpdateEvent);
  diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));
}
 
Example 13
Source Project: incubator-tez   Source File: TestTaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  conf = new Configuration();
  taskAttemptListener = mock(TaskAttemptListener.class);
  taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  credentials = new Credentials();
  clock = new SystemClock();
  locationHint = new TaskLocationHint(null, null);

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  dagId = TezDAGID.getInstance(appId, 1);
  vertexId = TezVertexID.getInstance(dagId, 1);
  appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
  mockContainerId = mock(ContainerId.class);
  mockContainer = mock(Container.class);
  mockAMContainer = mock(AMContainer.class);
  mockNodeId = mock(NodeId.class);
  when(mockContainer.getId()).thenReturn(mockContainerId);
  when(mockContainer.getNodeId()).thenReturn(mockNodeId);
  when(mockAMContainer.getContainer()).thenReturn(mockContainer);
  when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
  taskResource = Resource.newInstance(1024, 1);
  localResources = new HashMap<String, LocalResource>();
  environment = new HashMap<String, String>();
  javaOpts = "";
  leafVertex = false;
  containerContext = new ContainerContext(localResources, credentials,
      environment, javaOpts);
  Vertex vertex = mock(Vertex.class);
  eventHandler = new TestEventHandler();
  
  mockTask = new MockTaskImpl(vertexId, partition,
      eventHandler, conf, taskAttemptListener, clock,
      taskHeartbeatHandler, appContext, leafVertex, locationHint,
      taskResource, containerContext, vertex);
}
 
Example 14
Source Project: hadoop   Source File: TestTaskHeartbeatHandler.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTimeout() throws InterruptedException {
  EventHandler mockHandler = mock(EventHandler.class);
  Clock clock = new SystemClock();
  TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1);
  
  
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.TASK_TIMEOUT, 10); //10 ms
  conf.setInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 10); //10 ms
  
  hb.init(conf);
  hb.start();
  try {
    ApplicationId appId = ApplicationId.newInstance(0l, 5);
    JobId jobId = MRBuilderUtils.newJobId(appId, 4);
    TaskId tid = MRBuilderUtils.newTaskId(jobId, 3, TaskType.MAP);
    TaskAttemptId taid = MRBuilderUtils.newTaskAttemptId(tid, 2);
    hb.register(taid);
    Thread.sleep(100);
    //Events only happen when the task is canceled
    verify(mockHandler, times(2)).handle(any(Event.class));
  } finally {
    hb.stop();
  }
}
 
Example 15
Source Project: tez   Source File: TestRecoveryParser.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testRecoverableNonSummary2() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(mockDAGImpl.getID()).thenReturn(dagID);

  // MockRecoveryService will skip the non-summary event
  MockRecoveryService rService = new MockRecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write a DAGSubmittedEvent first to initialize summaryStream
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // It should be fine to skip other events, just for testing.
  TezVertexID vertexId = TezVertexID.getInstance(dagID, 0);
  rService.handle(new DAGHistoryEvent(dagID,
      new VertexGroupCommitStartedEvent(dagID, "group_1", 
          Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L)));
  rService.handle(new DAGHistoryEvent(dagID,
      new VertexGroupCommitFinishedEvent(dagID, "group_1", 
          Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L)));
  rService.stop();

  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertTrue(dagData.nonRecoverable);
  assertTrue(dagData.reason.contains("Vertex has been committed as member of vertex group"
            + ", but its full recovery events are not seen"));
}
 
Example 16
Source Project: big-c   Source File: TestRecovery.java    License: Apache License 2.0 5 votes vote down vote up
private MapTaskImpl getMockMapTask(long clusterTimestamp, EventHandler eh) {

    ApplicationId appId = ApplicationId.newInstance(clusterTimestamp, 1);
    JobId jobId = MRBuilderUtils.newJobId(appId, 1);

    int partitions = 2;

    Path remoteJobConfFile = mock(Path.class);
    JobConf conf = new JobConf();
    TaskAttemptListener taskAttemptListener = mock(TaskAttemptListener.class);
    Token<JobTokenIdentifier> jobToken =
        (Token<JobTokenIdentifier>) mock(Token.class);
    Credentials credentials = null;
    Clock clock = new SystemClock();
    int appAttemptId = 3;
    MRAppMetrics metrics = mock(MRAppMetrics.class);
    Resource minContainerRequirements = mock(Resource.class);
    when(minContainerRequirements.getMemory()).thenReturn(1000);

    ClusterInfo clusterInfo = mock(ClusterInfo.class);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getClusterInfo()).thenReturn(clusterInfo);

    TaskSplitMetaInfo taskSplitMetaInfo = mock(TaskSplitMetaInfo.class);
    MapTaskImpl mapTask = new MapTaskImpl(jobId, partitions,
        eh, remoteJobConfFile, conf,
        taskSplitMetaInfo, taskAttemptListener, jobToken, credentials, clock,
        appAttemptId, metrics, appContext);
    return mapTask;
  }
 
Example 17
Source Project: tez   Source File: TestRecoveryParser.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testRecoverableSummary_DAGFinishCommitting() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(mockDAGImpl.getID()).thenReturn(dagID);

  RecoveryService rService = new RecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write a DAGSubmittedEvent first to initialize summaryStream
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // It should be fine to skip other events, just for testing.
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGCommitStartedEvent(dagID, 0L)));
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGFinishedEvent(dagID, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null,
          appAttemptId, dagPlan)));
  rService.stop();

  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertEquals(dagID, dagData.recoveredDagID);
  assertEquals(DAGState.FAILED, dagData.dagState);
  assertFalse(dagData.nonRecoverable);
  assertNull(dagData.reason);
  assertTrue(dagData.isCompleted);
}
 
Example 18
Source Project: Bats   Source File: StreamingContainerManager.java    License: Apache License 2.0 4 votes vote down vote up
public StreamingContainerManager(LogicalPlan dag)
{
  this(dag, false, new SystemClock());
}
 
Example 19
Source Project: XLearning   Source File: JobHistoryServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  FileSystem fs;
  Configuration conf = new XLearningConfiguration();
  Path historyLog = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR));
  Path eventLog = new Path(conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR,
      XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR));
  int monitorInterval = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DELETE_MONITOR_TIME_INTERVAL);
  int logMaxAge = conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_LOG_MAX_AGE_MS,
      XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_MAX_AGE_MS);
  final Clock clock = new SystemClock();
  while (!Thread.currentThread().isInterrupted()) {
    try {
      LOG.info("Start delete log thread.");
      Long currentClock = clock.getTime();
      fs = FileSystem.get(conf);
      FileStatus[] allHistoryLog = fs.listStatus(historyLog);
      LOG.info("historyLog:" + historyLog);
      for (FileStatus historyLogPer : allHistoryLog) {
        LOG.info(historyLogPer.getPath() + ":" + String.valueOf(currentClock - historyLogPer.getModificationTime()));
        if ((currentClock - historyLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(historyLogPer.getPath());
        }
      }
      FileStatus[] allEventLog = fs.listStatus(eventLog);
      LOG.info("eventLog:" + eventLog);
      for (FileStatus eventLogPer : allEventLog) {
        LOG.info(eventLogPer.getPath() + ":" + String.valueOf(currentClock - eventLogPer.getModificationTime()));
        if ((currentClock - eventLogPer.getModificationTime()) > logMaxAge) {
          fs.delete(eventLogPer.getPath());
        }
      }

      Thread.sleep(monitorInterval);
    } catch (Exception e) {
      LOG.info("HistoryLog delete thread interrupted. " + e);
      break;
    }
  }
}
 
Example 20
Source Project: XLearning   Source File: ApplicationContainerListener.java    License: Apache License 2.0 4 votes vote down vote up
public ApplicationContainerListener(ApplicationContext applicationContext, Configuration conf) {
  super(ApplicationContainerListener.class.getSimpleName());
  this.setConfig(conf);
  this.containerId2Role = new ConcurrentHashMap<>();
  this.containerId2Status = new ConcurrentHashMap<>();
  this.reporterProgress = new ConcurrentHashMap<>();
  this.mapedTaskID = new ConcurrentHashMap<>();
  this.containersAppStartTimeMap = new ConcurrentHashMap<>();
  this.containersAppFinishTimeMap = new ConcurrentHashMap<>();
  this.single = conf.getBoolean(XLearningConfiguration.XLEARNING_MODE_SINGLE, XLearningConfiguration.DEFAULT_XLEARNING_MODE_SINGLE);
  this.clusterDef = new ConcurrentHashMap<>();
  this.clusterDef.put(XLearningConstants.WORKER, Collections.synchronizedList(new ArrayList<ContainerHostPair>()));
  this.clusterDef.put(XLearningConstants.PS, Collections.synchronizedList(new ArrayList<ContainerHostPair>()));
  this.clusterDef.put(XLearningConstants.EVALUATOR, Collections.synchronizedList(new ArrayList<ContainerHostPair>()));
  this.clusterDefStr = null;
  this.tfDistributionStrategy = conf.getBoolean(XLearningConfiguration.XLEARNING_TF_DISTRIBUTION_STRATEGY, XLearningConfiguration.DEFAULT_XLEARNING_TF_DISTRIBUTION_STRATEGY);
  this.tfEvaluator = conf.getBoolean(XLearningConfiguration.XLEARNING_TF_EVALUATOR, XLearningConfiguration.DEFAULT_XLEARNING_TF_EVALUATOR);
  this.lightGBMIpPortMap = new ConcurrentHashMap<>();
  this.lightGBMIpPortStr = null;
  this.lightLDAIpPortMap = new ConcurrentHashMap<>();
  this.lightLDAIpPortStr = null;
  this.applicationContext = applicationContext;
  this.clock = new SystemClock();
  this.runningContainers = new ConcurrentHashMap<>();
  this.containerTimeOut = conf.getInt(XLearningConfiguration.XLEARNING_TASK_TIMEOUT, XLearningConfiguration.DEFAULT_XLEARNING_TASK_TIMEOUT);
  this.localResourceTimeOut = conf.getInt(XLearningConfiguration.XLEARNING_LOCALRESOURCE_TIMEOUT, XLearningConfiguration.DEFAULT_XLEARNING_LOCALRESOURCE_TIMEOUT);
  this.monitorInterval = conf.getInt(XLearningConfiguration.XLEARNING_TASK_TIMEOUT_CHECK_INTERVAL_MS, XLearningConfiguration.DEFAULT_XLEARNING_TASK_TIMEOUT_CHECK_INTERVAL_MS);
  this.isXLearningTrainFinished = false;
  this.tensorboardUrl = null;
  this.isSaveInnerModel = false;
  this.interResultTimeStamp = Long.MIN_VALUE;
  this.containerId2InnerModel = new ConcurrentHashMap<>();
  this.containersCpuMetrics = new ConcurrentHashMap<>();
  this.containersCpuStatistics = new ConcurrentHashMap<>();
  if (System.getenv().containsKey(XLearningConstants.Environment.XLEARNING_APP_TYPE.toString())) {
    xlearningAppType = System.getenv(XLearningConstants.Environment.XLEARNING_APP_TYPE.toString()).toUpperCase();
  } else {
    xlearningAppType = "XLEARNING";
  }
  this.isAMFinished = -1;
}
 
Example 21
Source Project: tajo   Source File: SessionLivelinessMonitor.java    License: Apache License 2.0 4 votes vote down vote up
public SessionLivelinessMonitor(Dispatcher d) {
  super(SessionLivelinessMonitor.class.getSimpleName(), new SystemClock());
  this.dispatcher = d.getEventHandler();
}
 
Example 22
Source Project: incubator-tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
// Ensure ContainerTerminating and ContainerTerminated is handled correctly by
// the TaskAttempt
public void testContainerTerminatedAfterSuccess() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
  TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);

  TaskLocationHint locationHint = new TaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[] {"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AppContext appCtx = mock(AppContext.class);
  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mock(TaskHeartbeatHandler.class), appCtx, locationHint, false,
      resource, createFakeContainerContext(), false);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
  // At state STARTING.
  taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
      null));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.RUNNING);

  int expectedEventsAtRunning = 3;
  verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());

  taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));

  assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(),
      TaskAttemptState.SUCCEEDED);

  assertEquals(0, taImpl.getDiagnostics().size());

  int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());

  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);

  taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
      "Terminated"));
  int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());

  // Verify that the diagnostic message included in the Terminated event is not
  // captured - TA already succeeded.
  assertEquals(0, taImpl.getDiagnostics().size());
}
 
Example 23
Source Project: hadoop   Source File: ContainerAllocationExpirer.java    License: Apache License 2.0 4 votes vote down vote up
public ContainerAllocationExpirer(Dispatcher d) {
  super(ContainerAllocationExpirer.class.getName(), new SystemClock());
  this.dispatcher = d.getEventHandler();
}
 
Example 24
Source Project: hadoop   Source File: AMLivelinessMonitor.java    License: Apache License 2.0 4 votes vote down vote up
public AMLivelinessMonitor(Dispatcher d) {
  super("AMLivelinessMonitor", new SystemClock());
  this.dispatcher = d.getEventHandler();
}
 
Example 25
Source Project: tez   Source File: TestRecoveryParser.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testLastCorruptedRecoveryRecord() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
  when(appContext.getApplicationID()).thenReturn(appId);

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write data in attempt_1
  RecoveryService rService = new RecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // wait until DAGSubmittedEvent is handled in the RecoveryEventHandling thread
  rService.await();
  rService.outputStreamMap.get(dagID).write("INVALID_DATA".getBytes("UTF-8"));
  rService.stop();

  // write data in attempt_2
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/2"));
  rService = new RecoveryService(appContext);
  rService.init(conf);
  rService.start();
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
  rService.await();
  rService.outputStreamMap.get(dagID).write("INVALID_DATA".getBytes("UTF-8"));
  rService.stop();

  // corrupted last records will be skipped but the whole recovery logs will be read
  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertEquals(false, dagData.isCompleted);
  assertEquals(null, dagData.reason);
  assertEquals(false, dagData.nonRecoverable);
  // verify DAGSubmitedEvent & DAGInititlizedEvent is handled.
  verify(mockAppMaster).createDAG(any(DAGPlan.class),any(TezDAGID.class));
  assertNotNull(dagData.getDAGInitializedEvent());
}
 
Example 26
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testHeartbeatHandler() throws Exception {
  LOG.info("Running testHeartbeatHandler");

  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS, 1);
  ControlledClock clock = new ControlledClock(new SystemClock());
  AppContext appContext = mock(AppContext.class);
  when(appContext.getClock()).thenReturn(clock);
  when(appContext.getApplicationID()).thenReturn(
      ApplicationId.newInstance(1, 1));

  RMContainerAllocator allocator = new RMContainerAllocator(
      mock(ClientService.class), appContext) {
        @Override
        protected void register() {
        }
        @Override
        protected ApplicationMasterProtocol createSchedulerProxy() {
          return mock(ApplicationMasterProtocol.class);
        }
        @Override
        protected synchronized void heartbeat() throws Exception {
        }
  };
  allocator.init(conf);
  allocator.start();

  clock.setTime(5);
  int timeToWaitMs = 5000;
  while (allocator.getLastHeartbeatTime() != 5 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals(5, allocator.getLastHeartbeatTime());
  clock.setTime(7);
  timeToWaitMs = 5000;
  while (allocator.getLastHeartbeatTime() != 7 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals(7, allocator.getLastHeartbeatTime());

  final AtomicBoolean callbackCalled = new AtomicBoolean(false);
  allocator.runOnNextHeartbeat(new Runnable() {
    @Override
    public void run() {
      callbackCalled.set(true);
    }
  });
  clock.setTime(8);
  timeToWaitMs = 5000;
  while (allocator.getLastHeartbeatTime() != 8 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals(8, allocator.getLastHeartbeatTime());
  Assert.assertTrue(callbackCalled.get());
}
 
Example 27
Source Project: tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testCompletedAtSubmitted() throws ServicePluginException {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);

  locationHint = TaskLocationHint.createTaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[]{"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  @SuppressWarnings("deprecation")
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container, 0, 0, 0);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mockHeartbeatHandler, appCtx, false,
      resource, createFakeContainerContext(), false);
  TezTaskAttemptID taskAttemptID = taImpl.getID();
  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
  taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.STARTING);

  verify(mockHeartbeatHandler).register(taskAttemptID);

  int expectedEventsAtStarting = 4;
  verify(eventHandler, times(expectedEventsAtStarting)).handle(arg.capture());

  // Ensure status_updates are handled in the submitted state.
  taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID,
      new TaskStatusUpdateEvent(null, 0.1f, null, false)));

  taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));

  assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(),
      TaskAttemptState.SUCCEEDED);
  verify(mockHeartbeatHandler).unregister(taskAttemptID);
  assertEquals(0, taImpl.getDiagnostics().size());

  int expectedEvenstAfterTerminating = expectedEventsAtStarting + 3;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());


  Event e = verifyEventType(
      arg.getAllValues().subList(expectedEventsAtStarting,
          expectedEvenstAfterTerminating), TaskEventTASucceeded.class, 1);
  assertEquals(TaskEventType.T_ATTEMPT_SUCCEEDED, e.getType());
  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtStarting,
          expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtStarting,
          expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1);
}
 
Example 28
public ProportionalCapacityPreemptionPolicy(Configuration config,
    EventHandler<ContainerPreemptEvent> dispatcher,
    CapacityScheduler scheduler) {
  this(config, dispatcher, scheduler, new SystemClock());
}
 
Example 29
Source Project: big-c   Source File: TestTaskAttemptContainerRequest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAttemptContainerRequest() throws Exception {
  final Text SECRET_KEY_ALIAS = new Text("secretkeyalias");
  final byte[] SECRET_KEY = ("secretkey").getBytes();
  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(1);
  acls.put(ApplicationAccessType.VIEW_APP, "otheruser");
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  Path jobFile = mock(Path.class);

  EventHandler eventHandler = mock(EventHandler.class);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");

  // setup UGI for security so tokens and keys are preserved
  jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(jobConf);

  Credentials credentials = new Credentials();
  credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY);
  Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
      ("tokenid").getBytes(), ("tokenpw").getBytes(),
      new Text("tokenkind"), new Text("tokenservice"));

  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          mock(TaskSplitMetaInfo.class), jobConf, taListener,
          jobToken, credentials,
          new SystemClock(), null);

  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString());

  ContainerLaunchContext launchCtx =
      TaskAttemptImpl.createContainerLaunchContext(acls,
          jobConf, jobToken, taImpl.createRemoteTask(),
          TypeConverter.fromYarn(jobId),
          mock(WrappedJvmID.class), taListener,
          credentials);

  Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs());
  Credentials launchCredentials = new Credentials();

  DataInputByteBuffer dibb = new DataInputByteBuffer();
  dibb.reset(launchCtx.getTokens());
  launchCredentials.readTokenStorageStream(dibb);

  // verify all tokens specified for the task attempt are in the launch context
  for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
    Token<? extends TokenIdentifier> launchToken =
        launchCredentials.getToken(token.getService());
    Assert.assertNotNull("Token " + token.getService() + " is missing",
        launchToken);
    Assert.assertEquals("Token " + token.getService() + " mismatch",
        token, launchToken);
  }

  // verify the secret key is in the launch context
  Assert.assertNotNull("Secret key missing",
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS));
  Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY,
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS)));
}
 
Example 30
Source Project: incubator-tajo   Source File: TajoMaster.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void init(Configuration _conf) {
  this.systemConf = (TajoConf) _conf;

  context = new MasterContext(systemConf);
  clock = new SystemClock();

  try {
    RackResolver.init(systemConf);

    initResourceManager();
    initWebServer();

    this.dispatcher = new AsyncDispatcher();
    addIfService(dispatcher);

    // check the system directory and create if they are not created.
    checkAndInitializeSystemDirectories();
    this.storeManager = StorageManagerFactory.getStorageManager(systemConf);

    catalogServer = new CatalogServer(initBuiltinFunctions());
    addIfService(catalogServer);
    catalog = new LocalCatalogWrapper(catalogServer, systemConf);

    globalEngine = new GlobalEngine(context);
    addIfService(globalEngine);

    queryJobManager = new QueryJobManager(context);
    addIfService(queryJobManager);

    tajoMasterClientService = new TajoMasterClientService(context);
    addIfService(tajoMasterClientService);

    tajoMasterService = new TajoMasterService(context);
    addIfService(tajoMasterService);

  } catch (Exception e) {
    LOG.error(e.getMessage(), e);
  }

  super.init(systemConf);

  LOG.info("Tajo Master is initialized.");
}