Java Code Examples for org.apache.hadoop.service.Service

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

@Override
public void serviceStop() throws Exception {
  try {
    synchronized (serviceMap) {
      for (Service service : serviceMap.values()) {
        if (service.getServiceState() == Service.STATE.STARTED) {
          service.unregisterServiceListener(this);
          service.stop();
        }
      }
      serviceMap.clear();
      serviceMetaData.clear();
    }
  } finally {
    super.serviceStop();
  }
}
 

@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 

@Test
public void testAuxServiceRecoverySetup() throws IOException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, TEST_DIR.toString());
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      RecoverableServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      RecoverableServiceB.class, Service.class);
  try {
    final AuxServices aux = new AuxServices();
    aux.init(conf);
    Assert.assertEquals(2, aux.getServices().size());
    File auxStorageDir = new File(TEST_DIR,
        AuxServices.STATE_STORE_ROOT_NAME);
    Assert.assertEquals(2, auxStorageDir.listFiles().length);
    aux.close();
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 

@Test
public void testPolicyInitializeAfterSchedulerInitialized() {
  @SuppressWarnings("resource")
  MockRM rm = new MockRM(conf);
  rm.init(conf);
  
  // ProportionalCapacityPreemptionPolicy should be initialized after
  // CapacityScheduler initialized. We will 
  // 1) find SchedulingMonitor from RMActiveService's service list, 
  // 2) check if ResourceCalculator in policy is null or not. 
  // If it's not null, we can come to a conclusion that policy initialized
  // after scheduler got initialized
  for (Service service : rm.getRMActiveService().getServices()) {
    if (service instanceof SchedulingMonitor) {
      ProportionalCapacityPreemptionPolicy policy =
          (ProportionalCapacityPreemptionPolicy) ((SchedulingMonitor) service)
              .getSchedulingEditPolicy();
      assertNotNull(policy.getResourceCalculator());
      return;
    }
  }
  
  fail("Failed to find SchedulingMonitor service, please check what happened");
}
 

@Test
public void testPolicyInitializeAfterSchedulerInitialized() {
  @SuppressWarnings("resource")
  MockRM rm = new MockRM(conf);
  rm.init(conf);
  
  // ProportionalCapacityPreemptionPolicy should be initialized after
  // CapacityScheduler initialized. We will 
  // 1) find SchedulingMonitor from RMActiveService's service list, 
  // 2) check if ResourceCalculator in policy is null or not. 
  // If it's not null, we can come to a conclusion that policy initialized
  // after scheduler got initialized
  for (Service service : rm.getRMActiveService().getServices()) {
    if (service instanceof SchedulingMonitor) {
      ProportionalCapacityPreemptionPolicy policy =
          (ProportionalCapacityPreemptionPolicy) ((SchedulingMonitor) service)
              .getSchedulingEditPolicy();
      assertNotNull(policy.getResourceCalculator());
      return;
    }
  }
  
  fail("Failed to find SchedulingMonitor service, please check what happened");
}
 

/**
 * Show that if the service failed during an init
 * operation, stop was called.
 */

@Test
public void testStopFailingInitAndStop() throws Throwable {
  BreakableService svc = new BreakableService(true, false, true);
  svc.registerServiceListener(new LoggingStateChangeListener());
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    assertEquals(Service.STATE.INITED, e.state);
  }
  //the service state is stopped
  assertServiceStateStopped(svc);
  assertEquals(Service.STATE.INITED, svc.getFailureState());

  Throwable failureCause = svc.getFailureCause();
  assertNotNull("Null failure cause in " + svc, failureCause);
  BreakableService.BrokenLifecycleEvent cause =
    (BreakableService.BrokenLifecycleEvent) failureCause;
  assertNotNull("null state in " + cause + " raised by " + svc, cause.state);
  assertEquals(Service.STATE.INITED, cause.state);
}
 

public void waitForState(Service.STATE finalState) throws Exception {
  if (finalState == Service.STATE.STOPPED) {
     Assert.assertTrue("Timeout while waiting for MRApp to stop",
         waitForServiceToStop(20 * 1000));
  } else {
    int timeoutSecs = 0;
    while (!finalState.equals(getServiceState()) && timeoutSecs++ < 20) {
      System.out.println("MRApp State is : " + getServiceState()
          + " Waiting for state : " + finalState);
      Thread.sleep(500);
    }
    System.out.println("MRApp State is : " + getServiceState());
    Assert.assertEquals("MRApp state is not correct (timedout)", finalState,
        getServiceState());
  }
}
 

public void testDeletionofStagingOnKillLastTry() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
  appMaster.init(conf);
  assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  assertTrue("MRAppMaster isn't stopped",
             appMaster.isInState(Service.STATE.STOPPED));
  verify(fs).delete(stagingJobPath, true);
}
 

/**
 * Test that the {@link BreakableStateChangeListener} is picking up
 * the state changes and that its last event field is as expected.
 */
@Test
public void testEventHistory() {
  register();
  BreakableService service = new BreakableService();
  assertListenerState(listener, Service.STATE.NOTINITED);
  assertEquals(0, listener.getEventCount());
  service.init(new Configuration());
  assertListenerState(listener, Service.STATE.INITED);
  assertSame(service, listener.getLastService());
  assertListenerEventCount(listener, 1);

  service.start();
  assertListenerState(listener, Service.STATE.STARTED);
  assertListenerEventCount(listener, 2);

  service.stop();
  assertListenerState(listener, Service.STATE.STOPPED);
  assertListenerEventCount(listener, 3);
}
 

/**
 * Tests that the YarnClient is only shut down if it is not shared.
 */
@Test
public void testYarnClientShutDown() {
	YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor();

	yarnClusterDescriptor.close();

	assertTrue(yarnClient.isInState(Service.STATE.STARTED));

	final YarnClient closableYarnClient = YarnClient.createYarnClient();
	closableYarnClient.init(yarnConfiguration);
	closableYarnClient.start();

	yarnClusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(
			temporaryFolder.getRoot().getAbsolutePath(),
			new Configuration(),
			yarnConfiguration,
			closableYarnClient,
			false);

	yarnClusterDescriptor.close();

	assertTrue(closableYarnClient.isInState(Service.STATE.STOPPED));
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  // init timeline services first
  timelineStore = createTimelineStore(conf);
  addIfService(timelineStore);
  secretManagerService = createTimelineDelegationTokenSecretManagerService(conf);
  addService(secretManagerService);
  timelineDataManager = createTimelineDataManager(conf);
  addService(timelineDataManager);

  // init generic history service afterwards
  aclsManager = createApplicationACLsManager(conf);
  historyManager = createApplicationHistoryManager(conf);
  ahsClientService = createApplicationHistoryClientService(historyManager);
  addService(ahsClientService);
  addService((Service) historyManager);

  DefaultMetricsSystem.initialize("ApplicationHistoryServer");
  JvmMetrics.initSingleton("ApplicationHistoryServer", null);
  super.serviceInit(conf);
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

/**
 * Walk the {@link BreakableService} through it's lifecycle, 
 * more to verify that service's counters work than anything else
 * @throws Throwable if necessary
 */
@Test
public void testWalkthrough() throws Throwable {

  BreakableService svc = new BreakableService();
  assertServiceStateCreated(svc);
  assertStateCount(svc, Service.STATE.NOTINITED, 1);
  assertStateCount(svc, Service.STATE.INITED, 0);
  assertStateCount(svc, Service.STATE.STARTED, 0);
  assertStateCount(svc, Service.STATE.STOPPED, 0);
  svc.init(new Configuration());
  assertServiceStateInited(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  svc.start();
  assertServiceStateStarted(svc);
  assertStateCount(svc, Service.STATE.STARTED, 1);
  svc.stop();
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

/**
 * Show that if the service failed during an init
 * operation, it stays in the created state, even after stopping it
 * @throws Throwable if necessary
 */

@Test
public void testStopFailedInit() throws Throwable {
  BreakableService svc = new BreakableService(true, false, false);
  assertServiceStateCreated(svc);
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    //expected
  }
  //the service state wasn't passed
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
  //now try to stop
  svc.stop();
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

/**
 * Show that if the service failed during an init
 * operation, stop was called.
 */

@Test
public void testStopFailingInitAndStop() throws Throwable {
  BreakableService svc = new BreakableService(true, false, true);
  svc.registerServiceListener(new LoggingStateChangeListener());
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    assertEquals(Service.STATE.INITED, e.state);
  }
  //the service state is stopped
  assertServiceStateStopped(svc);
  assertEquals(Service.STATE.INITED, svc.getFailureState());

  Throwable failureCause = svc.getFailureCause();
  assertNotNull("Null failure cause in " + svc, failureCause);
  BreakableService.BrokenLifecycleEvent cause =
    (BreakableService.BrokenLifecycleEvent) failureCause;
  assertNotNull("null state in " + cause + " raised by " + svc, cause.state);
  assertEquals(Service.STATE.INITED, cause.state);
}
 

@Test
public void testServiceFailingNotifications() throws Throwable {
  BreakableService svc = new BreakableService(false, false, false);
  BreakableStateChangeListener listener = new BreakableStateChangeListener();
  listener.setFailingState(Service.STATE.STARTED);
  svc.registerServiceListener(listener);
  svc.init(new Configuration());
  assertEventCount(listener, 1);
  //start this; the listener failed but this won't show
  svc.start();
  //counter went up
  assertEventCount(listener, 2);
  assertEquals(1, listener.getFailureCount());
  //stop the service -this doesn't fail
  svc.stop();
  assertEventCount(listener, 3);
  assertEquals(1, listener.getFailureCount());
  svc.stop();
}
 

/**
 * Walk the {@link BreakableService} through it's lifecycle, 
 * more to verify that service's counters work than anything else
 * @throws Throwable if necessary
 */
@Test
public void testWalkthrough() throws Throwable {

  BreakableService svc = new BreakableService();
  assertServiceStateCreated(svc);
  assertStateCount(svc, Service.STATE.NOTINITED, 1);
  assertStateCount(svc, Service.STATE.INITED, 0);
  assertStateCount(svc, Service.STATE.STARTED, 0);
  assertStateCount(svc, Service.STATE.STOPPED, 0);
  svc.init(new Configuration());
  assertServiceStateInited(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  svc.start();
  assertServiceStateStarted(svc);
  assertStateCount(svc, Service.STATE.STARTED, 1);
  svc.stop();
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

public void testDeletionofStagingOnKillLastTry() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
  appMaster.init(conf);
  assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  assertTrue("MRAppMaster isn't stopped",
             appMaster.isInState(Service.STATE.STOPPED));
  verify(fs).delete(stagingJobPath, true);
}
 

@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 

/**
 * Assert that the breakable service has entered a state exactly the number
 * of time asserted.
 * @param service service -if null an assertion is raised.
 * @param state state to check.
 * @param expected expected count.
 */
public static void assertStateCount(BreakableService service,
                      Service.STATE state,
                      int expected) {
  assertNotNull("Null service", service);
  int actual = service.getCount(state);
  if (expected != actual) {
    fail("Expected entry count for state [" + state +"] of " + service
             + " to be " + expected + " but was " + actual);
  }
}
 

@Test
public void testStart() {
  assertEquals(STATE.INITED, webAppProxy.getServiceState());
  webAppProxy.start();
  for (Service service : webAppProxy.getServices()) {
    if (service instanceof WebAppProxy) {
      assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress);
    }
  }
  assertEquals(STATE.STARTED, webAppProxy.getServiceState());
}
 

@Override
protected void serviceStop() throws Exception {
  LOG.info("Stopping JobHistory");
  if (scheduledExecutor != null) {
    LOG.info("Stopping History Cleaner/Move To Done");
    scheduledExecutor.shutdown();
    boolean interrupted = false;
    long currentTime = System.currentTimeMillis();
    while (!scheduledExecutor.isShutdown()
        && System.currentTimeMillis() > currentTime + 1000l && !interrupted) {
      try {
        Thread.sleep(20);
      } catch (InterruptedException e) {
        interrupted = true;
      }
    }
    if (!scheduledExecutor.isShutdown()) {
      LOG.warn("HistoryCleanerService/move to done shutdown may not have " +
      		"succeeded, Forcing a shutdown");
      scheduledExecutor.shutdownNow();
    }
  }
  if (storage != null && storage instanceof Service) {
    ((Service) storage).stop();
  }
  if (hsManager != null) {
    hsManager.stop();
  }
  super.serviceStop();
}
 

private void resetDispatcher() {
  Dispatcher dispatcher = setupDispatcher();
  ((Service)dispatcher).init(this.conf);
  ((Service)dispatcher).start();
  removeService((Service)rmDispatcher);
  // Need to stop previous rmDispatcher before assigning new dispatcher
  // otherwise causes "AsyncDispatcher event handler" thread leak
  ((Service) rmDispatcher).stop();
  rmDispatcher = dispatcher;
  addIfService(rmDispatcher);
  rmContext.setDispatcher(rmDispatcher);
}
 

public void setClusterInfo(ClusterInfo clusterInfo) {
  // Only useful if set before a job is started.
  if (getServiceState() == Service.STATE.NOTINITED
      || getServiceState() == Service.STATE.INITED) {
    this.clusterInfo = clusterInfo;
  } else {
    throw new IllegalStateException(
        "ClusterInfo can only be set before the App is STARTED");
  }
}
 

private MockRM initRM(Configuration conf) {
  MockRM rm = new MockRM(conf) {
    @Override
    public RMNodeLabelsManager createNodeLabelManager() {
      return lmgr;
    }
  };
  rm.getRMContext().setNodeLabelManager(lmgr);
  rm.start();
  Assert.assertEquals(Service.STATE.STARTED, rm.getServiceState());
  return rm;
}
 

@Override
public synchronized void stateChanged(Service service) {
  eventCount++;
  lastService = service;
  lastState = service.getServiceState();
  stateEventList.add(lastState);
  if (lastState == failingState) {
    failureCount++;
    throw new BreakableService.BrokenLifecycleEvent(service,
                                                    "Failure entering "
                                                    + lastState
                                                    + " for "
                                                    + service.getName());
  }
}
 

protected void addIfService(Object object, boolean addDispatcher) {
  if (object instanceof Service) {
    Service service = (Service) object;
    ServiceWithDependency sd = new ServiceWithDependency(service);
    services.put(service, sd);
    if(addDispatcher) {
      addIfServiceDependency(service, dispatcher);
    }
  }
}
 

@Test
public void testJobHistoryEventHandlerIsFirstServiceToStop() {
  MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
      .getClass().getName(), true);
  Configuration conf = new Configuration();
  app.init(conf);
  Service[] services = app.getServices().toArray(new Service[0]);
  // Verifying that it is the last to be added is same as verifying that it is
  // the first to be stopped. CompositeService related tests already validate
  // this.
  Assert.assertEquals("JobHistoryEventHandler",
      services[services.length - 1].getName());
}