Java Code Examples for org.apache.hadoop.yarn.event.AsyncDispatcher

The following examples show how to use org.apache.hadoop.yarn.event.AsyncDispatcher. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestNMWebServicesApps.java    License: Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example 2
Source Project: tajo   Source File: QueryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  try {
    this.dispatcher = new AsyncDispatcher();
    addService(this.dispatcher);

    this.dispatcher.register(QueryJobEvent.Type.class, new QueryJobManagerEventHandler());

    TajoConf tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
    this.historyCache = new LRUMap(tajoConf.getIntVar(TajoConf.ConfVars.HISTORY_QUERY_CACHE_SIZE));
  } catch (Exception e) {
    LOG.error("Failed to init service " + getName() + " by exception " + e, e);
  }

  super.serviceInit(conf);
}
 
Example 3
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example 4
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 5
Source Project: big-c   Source File: TestFairSchedulerEventLog.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
 
Example 6
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example 7
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testCommitJobFailsJob() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer fail and verify the job fails
  syncBarrier.await();
  assertJobState(job, JobStateInternal.FAILED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 8
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 9
Source Project: big-c   Source File: TestNMWebServicesApps.java    License: Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example 10
Source Project: big-c   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example 11
Source Project: tajo   Source File: QueryMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {

  this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
  this.manager = RpcClientManager.getInstance();
  this.rpcClientParams = RpcParameterFactory.get(this.systemConf);

  querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT);
  queryMasterContext = new QueryMasterContext(systemConf);

  clock = new SystemClock();
  finishedQueryMasterTasksCache = new LRUMap(systemConf.getIntVar(TajoConf.ConfVars.HISTORY_QUERY_CACHE_SIZE));

  this.dispatcher = new AsyncDispatcher();
  addIfService(dispatcher);

  globalPlanner = new GlobalPlanner(systemConf, workerContext);

  dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler());
  dispatcher.register(QueryStopEvent.EventType.class, new QueryStopEventHandler());
  super.serviceInit(conf);
  LOG.info("QueryMaster inited");
}
 
Example 12
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example 13
@Test
@AgentAttachmentRule.Enforce
public void ContainerResourceMonitoringModule_should_attach_to_isProcessTreeOverLimit() throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InstantiationException, InvocationTargetException {
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    final Header[] header = new Header[1];
    final Object[] event = new Object[1];
    ContainerResourceMonitoringTracer.initEventHandler((t, h, o) -> {
        header[0] = h;
        event[0] = o;
    });
    ClassFileTransformer classFileTransformer = new ContainerResourceMonitoringTracer.MemorySizeTracer().installOnByteBuddyAgent();

    try {
        ContainerExecutor exec = mock(ContainerExecutor.class);
        AsyncDispatcher dispatcher = mock(AsyncDispatcher.class);
        Context ctx = mock(Context.class);

        Class<?> clazz = classLoader.loadClass(ContainersMonitorImpl.class.getName());
        Method m = clazz.getDeclaredMethod("isProcessTreeOverLimit", String.class, long.class, long.class, long.class);
        Object inFormat = clazz.getConstructor(ContainerExecutor.class, AsyncDispatcher.class, Context.class).newInstance(exec, dispatcher, ctx);

        m.setAccessible(true);
        m.invoke(inFormat, "container_e600_1516870220739_0069_01_000032", 2000, 1000, 3000);

        assertNotNull(header[0]);
        assertNotNull(event[0]);

    } finally {
        ByteBuddyAgent.getInstrumentation().removeTransformer(classFileTransformer);
    }
}
 
Example 14
Source Project: hadoop   Source File: CommonNodeLabelsManager.java    License: Apache License 2.0 5 votes vote down vote up
protected void initDispatcher(Configuration conf) {
  // create async handler
  dispatcher = new AsyncDispatcher();
  AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
  asyncDispatcher.init(conf);
  asyncDispatcher.setDrainEventsOnStop();
}
 
Example 15
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testRebootedDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.isLastAMRetry()).thenReturn(true);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
  assertJobState(job, JobStateInternal.REBOOT);
  // return the external state as ERROR since this is last retry.
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.ERROR, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 
Example 16
Source Project: hadoop   Source File: TestContainersMonitor.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainerMonitorMemFlags() {
  ContainersMonitor cm = null;

  long expPmem = 8192 * 1024 * 1024l;
  long expVmem = (long) (expPmem * 2.1f);

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());
}
 
Example 17
Source Project: hadoop   Source File: SystemMetricsPublisher.java    License: Apache License 2.0 5 votes vote down vote up
public MultiThreadedDispatcher(int num) {
  super(MultiThreadedDispatcher.class.getName());
  for (int i = 0; i < num; ++i) {
    AsyncDispatcher dispatcher = createDispatcher();
    dispatchers.add(dispatcher);
    addIfService(dispatcher);
  }
}
 
Example 18
Source Project: tajo   Source File: TestExecutionBlockCursor.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  util = new TajoTestingCluster();
  util.startCatalogCluster();

  conf = util.getConfiguration();
  conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");

  catalog = util.getCatalogService();
  catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:!234/warehouse");
  catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
  TPCH tpch = new TPCH();
  tpch.loadSchemas();
  tpch.loadOutSchema();
  for (String table : tpch.getTableNames()) {
    TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
    TableDesc d = CatalogUtil.newTableDesc(
        IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
    TableStats stats = new TableStats();
    stats.setNumBytes(TPCH.tableVolumes.get(table));
    d.setStats(stats);
    catalog.createTable(d);
  }

  analyzer = new SQLAnalyzer();
  logicalPlanner = new LogicalPlanner(catalog, TablespaceManager.getInstance());
  optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance());

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  planner = new GlobalPlanner(conf, catalog);
}
 
Example 19
Source Project: tajo   Source File: TajoResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);

  AsyncDispatcher dispatcher = new AsyncDispatcher();
  addIfService(dispatcher);

  rmContext = new TajoRMContext(dispatcher);

  this.queryIdSeed = String.valueOf(System.currentTimeMillis());

  this.nodeLivelinessMonitor = new NodeLivelinessMonitor(this.rmContext.getDispatcher());
  addIfService(this.nodeLivelinessMonitor);

  // Register event handler for Workers
  rmContext.getDispatcher().register(NodeEventType.class, new WorkerEventDispatcher(rmContext));

  resourceTracker = new TajoResourceTracker(this, nodeLivelinessMonitor);
  addIfService(resourceTracker);

  String schedulerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_SCHEDULER_CLASS);
  scheduler = loadScheduler(schedulerClassName);
  LOG.info("Loaded resource scheduler : " + scheduler.getClass());
  addIfService(scheduler);
  rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);

  super.serviceInit(systemConf);
}
 
Example 20
Source Project: hadoop   Source File: RMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception{
  // create async handler
  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.register(RMStateStoreEventType.class, 
                      new ForwardingEventHandler());
  dispatcher.setDrainEventsOnStop();
  initInternal(conf);
}
 
Example 21
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringKillAbort() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void abortJob(JobContext jobContext, State state)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILL_ABORT);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 22
Source Project: tajo   Source File: TestSimpleScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  conf = new TajoConf();
  nodeResource = NodeResource.createResource(2000, 3);
  service = new CompositeService(TestSimpleScheduler.class.getSimpleName()) {

    @Override
    protected void serviceInit(Configuration conf) throws Exception {
      dispatcher = new AsyncDispatcher();
      addService(dispatcher);

      rmContext = new TajoRMContext(dispatcher);
      rmContext.getDispatcher().register(NodeEventType.class,
          new TajoResourceManager.WorkerEventDispatcher(rmContext));

      barrier = new Semaphore(0);
      scheduler = new MySimpleScheduler(rmContext, barrier);
      addService(scheduler);
      rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);

      for (int i = 0; i < workerNum; i++) {
        WorkerConnectionInfo conn = new WorkerConnectionInfo("host" + i, 28091 + i, 28092, 21000, 28093, 28080);
        rmContext.getNodes().putIfAbsent(conn.getId(),
            new NodeStatus(rmContext, NodeResources.clone(nodeResource), conn));
        rmContext.getDispatcher().getEventHandler().handle(new NodeEvent(conn.getId(), NodeEventType.STARTED));
      }
      super.serviceInit(conf);
    }
  };
  service.init(conf);
  service.start();

  assertEquals(workerNum, rmContext.getNodes().size());
  totalResource = NodeResources.createResource(0);
  for(NodeStatus nodeStatus : rmContext.getNodes().values()) {
    NodeResources.addTo(totalResource, nodeStatus.getTotalResourceCapability());
  }
}
 
Example 23
Source Project: tajo   Source File: QueryMasterTask.java    License: Apache License 2.0 5 votes vote down vote up
public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext,
                       QueryId queryId, Session session, QueryContext queryContext,
                       String jsonExpr, NodeResource allocation, AsyncDispatcher dispatcher) {

  super(QueryMasterTask.class.getName());
  this.queryMasterContext = queryMasterContext;
  this.queryId = queryId;
  this.session = session;
  this.queryContext = queryContext;
  this.jsonExpr = jsonExpr;
  this.allocation = allocation;
  this.querySubmitTime = System.currentTimeMillis();
  this.dispatcher = dispatcher;
}
 
Example 24
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testCheckJobCompleteSuccess() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer complete and verify the job succeeds
  syncBarrier.await();
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  job.handle(new JobEvent(job.getID(),
      JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.SUCCEEDED);

  job.handle(new JobEvent(job.getID(), 
      JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 25
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testRebootedDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.isLastAMRetry()).thenReturn(true);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
  assertJobState(job, JobStateInternal.REBOOT);
  // return the external state as ERROR since this is last retry.
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.ERROR, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 
Example 26
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringSetup() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void setupJob(JobContext jobContext)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 27
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testKilledDuringKillAbort() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void abortJob(JobContext jobContext, State state)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILL_ABORT);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 
Example 28
Source Project: incubator-tajo   Source File: QueryJobManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(Configuration conf) {
  try {
    this.dispatcher = new AsyncDispatcher();
    addService(this.dispatcher);

    this.dispatcher.register(QueryJobEvent.Type.class, new QueryJobManagerEventHandler());
  } catch (Exception e) {
    catchException(null, e);
  }

  super.init(conf);
}
 
Example 29
Source Project: big-c   Source File: ContainersMonitorImpl.java    License: Apache License 2.0 5 votes vote down vote up
public ContainersMonitorImpl(ContainerExecutor exec,
    AsyncDispatcher dispatcher, Context context) {
  super("containers-monitor");

  this.containerExecutor = exec;
  this.eventDispatcher = dispatcher;
  this.context = context;

  this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
  this.containersToBeRemoved = new ArrayList<ContainerId>();
  this.monitoringThread = new MonitoringThread();
}
 
Example 30
Source Project: big-c   Source File: TestContainersMonitor.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainerMonitorMemFlags() {
  ContainersMonitor cm = null;

  long expPmem = 8192 * 1024 * 1024l;
  long expVmem = (long) (expPmem * 2.1f);

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());
}