org.apache.hadoop.yarn.event.AsyncDispatcher Java Examples

The following examples show how to use org.apache.hadoop.yarn.event.AsyncDispatcher. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestNMWebServicesContainers.java    From big-c with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #2
Source File: TestJobImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testCommitJobFailsJob() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer fail and verify the job fails
  syncBarrier.await();
  assertJobState(job, JobStateInternal.FAILED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #3
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #4
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #5
Source File: TestFairSchedulerEventLog.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
 
Example #6
Source File: QueryManager.java    From tajo with Apache License 2.0 6 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  try {
    this.dispatcher = new AsyncDispatcher();
    addService(this.dispatcher);

    this.dispatcher.register(QueryJobEvent.Type.class, new QueryJobManagerEventHandler());

    TajoConf tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
    this.historyCache = new LRUMap(tajoConf.getIntVar(TajoConf.ConfVars.HISTORY_QUERY_CACHE_SIZE));
  } catch (Exception e) {
    LOG.error("Failed to init service " + getName() + " by exception " + e, e);
  }

  super.serviceInit(conf);
}
 
Example #7
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example #8
Source File: TestNMWebServicesApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #9
Source File: TestNMWebServicesApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #10
Source File: TestJobImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #11
Source File: QueryMaster.java    From tajo with Apache License 2.0 6 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {

  this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
  this.manager = RpcClientManager.getInstance();
  this.rpcClientParams = RpcParameterFactory.get(this.systemConf);

  querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT);
  queryMasterContext = new QueryMasterContext(systemConf);

  clock = new SystemClock();
  finishedQueryMasterTasksCache = new LRUMap(systemConf.getIntVar(TajoConf.ConfVars.HISTORY_QUERY_CACHE_SIZE));

  this.dispatcher = new AsyncDispatcher();
  addIfService(dispatcher);

  globalPlanner = new GlobalPlanner(systemConf, workerContext);

  dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler());
  dispatcher.register(QueryStopEvent.EventType.class, new QueryStopEventHandler());
  super.serviceInit(conf);
  LOG.info("QueryMaster inited");
}
 
Example #12
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example #13
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringSetup() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void setupJob(JobContext jobContext)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #14
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringKillAbort() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void abortJob(JobContext jobContext, State state)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILL_ABORT);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #15
Source File: TestExecutionBlockCursor.java    From tajo with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  util = new TajoTestingCluster();
  util.startCatalogCluster();

  conf = util.getConfiguration();
  conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");

  catalog = util.getCatalogService();
  catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:!234/warehouse");
  catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
  TPCH tpch = new TPCH();
  tpch.loadSchemas();
  tpch.loadOutSchema();
  for (String table : tpch.getTableNames()) {
    TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
    TableDesc d = CatalogUtil.newTableDesc(
        IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
    TableStats stats = new TableStats();
    stats.setNumBytes(TPCH.tableVolumes.get(table));
    d.setStats(stats);
    catalog.createTable(d);
  }

  analyzer = new SQLAnalyzer();
  logicalPlanner = new LogicalPlanner(catalog, TablespaceManager.getInstance());
  optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance());

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  planner = new GlobalPlanner(conf, catalog);
}
 
Example #16
Source File: TestSimpleScheduler.java    From tajo with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  conf = new TajoConf();
  nodeResource = NodeResource.createResource(2000, 3);
  service = new CompositeService(TestSimpleScheduler.class.getSimpleName()) {

    @Override
    protected void serviceInit(Configuration conf) throws Exception {
      dispatcher = new AsyncDispatcher();
      addService(dispatcher);

      rmContext = new TajoRMContext(dispatcher);
      rmContext.getDispatcher().register(NodeEventType.class,
          new TajoResourceManager.WorkerEventDispatcher(rmContext));

      barrier = new Semaphore(0);
      scheduler = new MySimpleScheduler(rmContext, barrier);
      addService(scheduler);
      rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);

      for (int i = 0; i < workerNum; i++) {
        WorkerConnectionInfo conn = new WorkerConnectionInfo("host" + i, 28091 + i, 28092, 21000, 28093, 28080);
        rmContext.getNodes().putIfAbsent(conn.getId(),
            new NodeStatus(rmContext, NodeResources.clone(nodeResource), conn));
        rmContext.getDispatcher().getEventHandler().handle(new NodeEvent(conn.getId(), NodeEventType.STARTED));
      }
      super.serviceInit(conf);
    }
  };
  service.init(conf);
  service.start();

  assertEquals(workerNum, rmContext.getNodes().size());
  totalResource = NodeResources.createResource(0);
  for(NodeStatus nodeStatus : rmContext.getNodes().values()) {
    NodeResources.addTo(totalResource, nodeStatus.getTotalResourceCapability());
  }
}
 
Example #17
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testRebootedDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.isLastAMRetry()).thenReturn(true);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
  assertJobState(job, JobStateInternal.REBOOT);
  // return the external state as ERROR since this is last retry.
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.ERROR, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 
Example #18
Source File: QueryMasterTask.java    From tajo with Apache License 2.0 5 votes vote down vote up
public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext,
                       QueryId queryId, Session session, QueryContext queryContext,
                       String jsonExpr, NodeResource allocation, AsyncDispatcher dispatcher) {

  super(QueryMasterTask.class.getName());
  this.queryMasterContext = queryMasterContext;
  this.queryId = queryId;
  this.session = session;
  this.queryContext = queryContext;
  this.jsonExpr = jsonExpr;
  this.allocation = allocation;
  this.querySubmitTime = System.currentTimeMillis();
  this.dispatcher = dispatcher;
}
 
Example #19
Source File: TajoResourceManager.java    From tajo with Apache License 2.0 5 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);

  AsyncDispatcher dispatcher = new AsyncDispatcher();
  addIfService(dispatcher);

  rmContext = new TajoRMContext(dispatcher);

  this.queryIdSeed = String.valueOf(System.currentTimeMillis());

  this.nodeLivelinessMonitor = new NodeLivelinessMonitor(this.rmContext.getDispatcher());
  addIfService(this.nodeLivelinessMonitor);

  // Register event handler for Workers
  rmContext.getDispatcher().register(NodeEventType.class, new WorkerEventDispatcher(rmContext));

  resourceTracker = new TajoResourceTracker(this, nodeLivelinessMonitor);
  addIfService(resourceTracker);

  String schedulerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_SCHEDULER_CLASS);
  scheduler = loadScheduler(schedulerClassName);
  LOG.info("Loaded resource scheduler : " + scheduler.getClass());
  addIfService(scheduler);
  rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);

  super.serviceInit(systemConf);
}
 
Example #20
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testCheckJobCompleteSuccess() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer complete and verify the job succeeds
  syncBarrier.await();
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  job.handle(new JobEvent(job.getID(),
      JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.SUCCEEDED);

  job.handle(new JobEvent(job.getID(), 
      JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #21
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testRebootedDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.isLastAMRetry()).thenReturn(true);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
  assertJobState(job, JobStateInternal.REBOOT);
  // return the external state as ERROR since this is last retry.
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.ERROR, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 
Example #22
Source File: CommonNodeLabelsManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void initDispatcher(Configuration conf) {
  // create async handler
  dispatcher = new AsyncDispatcher();
  AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
  asyncDispatcher.init(conf);
  asyncDispatcher.setDrainEventsOnStop();
}
 
Example #23
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringKillAbort() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void abortJob(JobContext jobContext, State state)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILL_ABORT);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #24
Source File: QueryJobManager.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Configuration conf) {
  try {
    this.dispatcher = new AsyncDispatcher();
    addService(this.dispatcher);

    this.dispatcher.register(QueryJobEvent.Type.class, new QueryJobManagerEventHandler());
  } catch (Exception e) {
    catchException(null, e);
  }

  super.init(conf);
}
 
Example #25
Source File: ContainersMonitorImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ContainersMonitorImpl(ContainerExecutor exec,
    AsyncDispatcher dispatcher, Context context) {
  super("containers-monitor");

  this.containerExecutor = exec;
  this.eventDispatcher = dispatcher;
  this.context = context;

  this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
  this.containersToBeRemoved = new ArrayList<ContainerId>();
  this.monitoringThread = new MonitoringThread();
}
 
Example #26
Source File: TestContainersMonitor.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainerMonitorMemFlags() {
  ContainersMonitor cm = null;

  long expPmem = 8192 * 1024 * 1024l;
  long expVmem = (long) (expPmem * 2.1f);

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());
}
 
Example #27
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 5 votes vote down vote up
public MultiThreadedDispatcher(int num) {
  super(MultiThreadedDispatcher.class.getName());
  for (int i = 0; i < num; ++i) {
    AsyncDispatcher dispatcher = createDispatcher();
    dispatchers.add(dispatcher);
    addIfService(dispatcher);
  }
}
 
Example #28
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RMContext mockRMContext(int n, long time) {
  final List<RMApp> apps = newRMApps(n, time, RMAppState.FINISHED);
  final ConcurrentMap<ApplicationId, RMApp> map = Maps.newConcurrentMap();
  for (RMApp app : apps) {
    map.put(app.getApplicationId(), app);
  }
  Dispatcher rmDispatcher = new AsyncDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = new ContainerAllocationExpirer(
      rmDispatcher);
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext context = new RMContextImpl(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      null, null, null, null, null, writer) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return map;
    }
  };
  ((RMContextImpl)context).setStateStore(mock(RMStateStore.class));
  metricsPublisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)context).setSystemMetricsPublisher(metricsPublisher);
  return context;
}
 
Example #29
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testCheckJobCompleteSuccess() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer complete and verify the job succeeds
  syncBarrier.await();
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  job.handle(new JobEvent(job.getID(),
      JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.SUCCEEDED);

  job.handle(new JobEvent(job.getID(), 
      JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  dispatcher.stop();
  commitHandler.stop();
}
 
Example #30
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}