Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
public ResourceLocalizationService(Dispatcher dispatcher,
    ContainerExecutor exec, DeletionService delService,
    LocalDirsHandlerService dirsHandler, Context context) {

  super(ResourceLocalizationService.class.getName());
  this.exec = exec;
  this.dispatcher = dispatcher;
  this.delService = delService;
  this.dirsHandler = dirsHandler;

  this.cacheCleanup = new ScheduledThreadPoolExecutor(1,
      new ThreadFactoryBuilder()
        .setNameFormat("ResourceLocalizationService Cache Cleanup")
        .build());
  this.stateStore = context.getNMStateStore();
  this.nmContext = context;
}
 
Example 2
Source Project: hadoop   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);

  this.olr = new OwnLocalResources();
}
 
Example 3
Source Project: hadoop   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example 4
Source Project: hadoop   Source File: TestDockerContainerRuntime.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testContainerLivelinessCheck()
    throws ContainerExecutionException, PrivilegedOperationException {

  DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
      mockExecutor, mockCGroupsHandler);
  builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
      .setExecutionAttribute(USER, user)
      .setExecutionAttribute(PID, signalPid)
          .setExecutionAttribute(SIGNAL, ContainerExecutor.Signal.NULL);
    runtime.initialize(getConfigurationWithMockContainerExecutor());
    runtime.signalContainer(builder.build());

  PrivilegedOperation op = capturePrivilegedOperation();
  Assert.assertEquals(op.getOperationType(),
      PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
  Assert.assertEquals("run_as_user", op.getArguments().get(0));
  Assert.assertEquals("user", op.getArguments().get(1));
  Assert.assertEquals("2", op.getArguments().get(2));
  Assert.assertEquals("1234", op.getArguments().get(3));
    Assert.assertEquals("0", op.getArguments().get(4));
}
 
Example 5
Source Project: hadoop   Source File: TestDockerContainerRuntime.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> getDockerCommandsForSignal(
    ContainerExecutor.Signal signal)
    throws ContainerExecutionException, PrivilegedOperationException,
    IOException {

  DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
      mockExecutor, mockCGroupsHandler);
  builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
      .setExecutionAttribute(USER, user)
      .setExecutionAttribute(PID, signalPid)
          .setExecutionAttribute(SIGNAL, signal);
    runtime.initialize(getConfigurationWithMockContainerExecutor());
  runtime.signalContainer(builder.build());

  PrivilegedOperation op = capturePrivilegedOperation();
  Assert.assertEquals(op.getOperationType(),
      PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
  String dockerCommandFile = op.getArguments().get(0);
  return Files.readAllLines(Paths.get(dockerCommandFile),
      Charset.forName("UTF-8"));
}
 
Example 6
Source Project: big-c   Source File: ResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
public ResourceLocalizationService(Dispatcher dispatcher,
    ContainerExecutor exec, DeletionService delService,
    LocalDirsHandlerService dirsHandler, Context context) {

  super(ResourceLocalizationService.class.getName());
  this.exec = exec;
  this.dispatcher = dispatcher;
  this.delService = delService;
  this.dirsHandler = dirsHandler;

  this.cacheCleanup = new ScheduledThreadPoolExecutor(1,
      new ThreadFactoryBuilder()
        .setNameFormat("ResourceLocalizationService Cache Cleanup")
        .build());
  this.stateStore = context.getNMStateStore();
  this.nmContext = context;
}
 
Example 7
Source Project: big-c   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
}
 
Example 8
Source Project: big-c   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example 9
@Test
@AgentAttachmentRule.Enforce
public void ContainerResourceMonitoringModule_should_attach_to_isProcessTreeOverLimit() throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InstantiationException, InvocationTargetException {
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    final Header[] header = new Header[1];
    final Object[] event = new Object[1];
    ContainerResourceMonitoringTracer.initEventHandler((t, h, o) -> {
        header[0] = h;
        event[0] = o;
    });
    ClassFileTransformer classFileTransformer = new ContainerResourceMonitoringTracer.MemorySizeTracer().installOnByteBuddyAgent();

    try {
        ContainerExecutor exec = mock(ContainerExecutor.class);
        AsyncDispatcher dispatcher = mock(AsyncDispatcher.class);
        Context ctx = mock(Context.class);

        Class<?> clazz = classLoader.loadClass(ContainersMonitorImpl.class.getName());
        Method m = clazz.getDeclaredMethod("isProcessTreeOverLimit", String.class, long.class, long.class, long.class);
        Object inFormat = clazz.getConstructor(ContainerExecutor.class, AsyncDispatcher.class, Context.class).newInstance(exec, dispatcher, ctx);

        m.setAccessible(true);
        m.invoke(inFormat, "container_e600_1516870220739_0069_01_000032", 2000, 1000, 3000);

        assertNotNull(header[0]);
        assertNotNull(event[0]);

    } finally {
        ByteBuddyAgent.getInstrumentation().removeTransformer(classFileTransformer);
    }
}
 
Example 10
Source Project: hadoop   Source File: RecoveredContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
public RecoveredContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager)
{
  super(context, configuration, dispatcher, exec, app, container, dirsHandler,
    containerManager);
  this.shouldLaunchContainer.set(true);
}
 
Example 11
Source Project: hadoop   Source File: ContainersLauncher.java    License: Apache License 2.0 5 votes vote down vote up
public ContainersLauncher(Context context, Dispatcher dispatcher,
    ContainerExecutor exec, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  super("containers-launcher");
  this.exec = exec;
  this.context = context;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
}
 
Example 12
Source Project: hadoop   Source File: ContainersMonitorImpl.java    License: Apache License 2.0 5 votes vote down vote up
public ContainersMonitorImpl(ContainerExecutor exec,
    AsyncDispatcher dispatcher, Context context) {
  super("containers-monitor");

  this.containerExecutor = exec;
  this.eventDispatcher = dispatcher;
  this.context = context;

  this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
  this.containersToBeRemoved = new ArrayList<ContainerId>();
  this.monitoringThread = new MonitoringThread();
}
 
Example 13
Source Project: hadoop   Source File: TestDockerContainerRuntime.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDockerStopOnTermSignal()
    throws ContainerExecutionException, PrivilegedOperationException,
    IOException {
  List<String> dockerCommands = getDockerCommandsForSignal(
      ContainerExecutor.Signal.TERM);
  Assert.assertEquals(1, dockerCommands.size());
  Assert.assertEquals("stop container_id", dockerCommands.get(0));
}
 
Example 14
Source Project: hadoop   Source File: TestDockerContainerRuntime.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDockerStopOnKillSignal()
    throws ContainerExecutionException, PrivilegedOperationException,
    IOException {
  List<String> dockerCommands = getDockerCommandsForSignal(
      ContainerExecutor.Signal.KILL);
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals("stop container_id", dockerCommands.get(0));
}
 
Example 15
Source Project: hadoop   Source File: TestDockerContainerRuntime.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDockerStopOnQuitSignal()
    throws ContainerExecutionException, PrivilegedOperationException,
    IOException {
  List<String> dockerCommands = getDockerCommandsForSignal(
          ContainerExecutor.Signal.QUIT);
  Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals("stop container_id", dockerCommands.get(0));
}
 
Example 16
Source Project: hadoop   Source File: TestContainersMonitor.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainerMonitorMemFlags() {
  ContainersMonitor cm = null;

  long expPmem = 8192 * 1024 * 1024l;
  long expVmem = (long) (expPmem * 2.1f);

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());
}
 
Example 17
Source Project: big-c   Source File: RecoveredContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
public RecoveredContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager)
{
  super(context, configuration, dispatcher, exec, app, container, dirsHandler,
    containerManager);
  this.shouldLaunchContainer.set(true);
}
 
Example 18
Source Project: big-c   Source File: ContainersLauncher.java    License: Apache License 2.0 5 votes vote down vote up
public ContainersLauncher(Context context, Dispatcher dispatcher,
    ContainerExecutor exec, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  super("containers-launcher");
  this.exec = exec;
  this.context = context;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
}
 
Example 19
Source Project: big-c   Source File: ContainersMonitorImpl.java    License: Apache License 2.0 5 votes vote down vote up
public ContainersMonitorImpl(ContainerExecutor exec,
    AsyncDispatcher dispatcher, Context context) {
  super("containers-monitor");

  this.containerExecutor = exec;
  this.eventDispatcher = dispatcher;
  this.context = context;

  this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
  this.containersToBeRemoved = new ArrayList<ContainerId>();
  this.monitoringThread = new MonitoringThread();
}
 
Example 20
Source Project: big-c   Source File: TestContainersMonitor.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainerMonitorMemFlags() {
  ContainersMonitor cm = null;

  long expPmem = 8192 * 1024 * 1024l;
  long expVmem = (long) (expPmem * 2.1f);

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, false, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(false, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(true, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(true, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());

  cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
      mock(AsyncDispatcher.class), mock(Context.class));
  cm.init(getConfForCM(false, true, 8192, 2.1f));
  assertEquals(expPmem, cm.getPmemAllocatedForContainers());
  assertEquals(expVmem, cm.getVmemAllocatedForContainers());
  assertEquals(false, cm.isPmemCheckEnabled());
  assertEquals(true, cm.isVmemCheckEnabled());
}
 
Example 21
Source Project: hadoop   Source File: DockerLinuxContainerRuntime.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void signalContainer(ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL);

  PrivilegedOperation privOp = null;
  // Handle liveliness checks, send null signal to pid
  if(ContainerExecutor.Signal.NULL.equals(signal)) {
    privOp = new PrivilegedOperation(
        PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
    privOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
        ctx.getExecutionAttribute(USER),
        Integer.toString(PrivilegedOperation.RunAsUserCommand
            .SIGNAL_CONTAINER.getValue()),
        ctx.getExecutionAttribute(PID),
        Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));

  // All other signals handled as docker stop
  } else {
    String containerId = ctx.getContainer().getContainerId().toString();
    DockerStopCommand stopCommand = new DockerStopCommand(containerId);

    String clientConfigDir =
        conf.get(YarnConfiguration.NM_DOCKER_CLIENT_CONFIG_DIRECTORY);
    stopCommand.setClientConfigDir(clientConfigDir);

    String commandFile = dockerClient.writeCommandToTempFile(stopCommand,
        containerId);
    privOp = new PrivilegedOperation(
        PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
    privOp.appendArgs(commandFile);
  }

  //Some failures here are acceptable. Let the calling executor decide.
  privOp.disableFailureLogging();

  try {
  privilegedOperationExecutor.executePrivilegedOperation(null,
      privOp, null, container.getLaunchContext().getEnvironment(),
      false);
  } catch (PrivilegedOperationException e) {
    throw new ContainerExecutionException("Signal container failed", e
        .getExitCode(), e.getOutput(), e.getErrorOutput());
  }
}
 
Example 22
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
public ContainerManagerImpl(Context context, ContainerExecutor exec,
    DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
    NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
    LocalDirsHandlerService dirsHandler) {
  super(ContainerManagerImpl.class.getName());
  this.context = context;
  this.dirsHandler = dirsHandler;

  // ContainerManager level dispatcher.
  dispatcher = new AsyncDispatcher();
  this.deletionService = deletionContext;
  this.metrics = metrics;

  rsrcLocalizationSrvc =
      createResourceLocalizationService(exec, deletionContext, context);
  addService(rsrcLocalizationSrvc);

  containersLauncher = createContainersLauncher(context, exec);
  addService(containersLauncher);

  this.nodeStatusUpdater = nodeStatusUpdater;
  this.aclsManager = aclsManager;

  // Start configurable services
  auxiliaryServices = new AuxServices();
  auxiliaryServices.registerServiceListener(this);
  addService(auxiliaryServices);

  this.containersMonitor =
      new ContainersMonitorImpl(exec, dispatcher, this.context);
  addService(this.containersMonitor);

  dispatcher.register(ContainerEventType.class,
      new ContainerEventDispatcher());
  dispatcher.register(ApplicationEventType.class,
      new ApplicationEventDispatcher());
  dispatcher.register(LocalizationEventType.class, rsrcLocalizationSrvc);
  dispatcher.register(AuxServicesEventType.class, auxiliaryServices);
  dispatcher.register(ContainersMonitorEventType.class, containersMonitor);
  dispatcher.register(ContainersLauncherEventType.class, containersLauncher);
  
  addService(dispatcher);

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();
}
 
Example 23
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
protected ResourceLocalizationService createResourceLocalizationService(
    ContainerExecutor exec, DeletionService deletionContext, Context context) {
  return new ResourceLocalizationService(this.dispatcher, exec,
      deletionContext, dirsHandler, context);
}
 
Example 24
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
protected ContainersLauncher createContainersLauncher(Context context,
    ContainerExecutor exec) {
  return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
}
 
Example 25
Source Project: hadoop   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 4 votes vote down vote up
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 
Example 26
Source Project: hadoop   Source File: TestContainerLaunch.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 20000)
public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    // echo "hello" to stdout and "error" to stderr and exit code with 2;
    String command = Shell.WINDOWS ?
        "@echo \"hello\" & @echo \"error\" 1>&2 & exit /b 2" :
        "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
    PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
    FileUtil.setExecutable(shellFile, true);
    writer.println(command);
    writer.close();
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    List<String> commands = new ArrayList<String>();
    commands.add(command);
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, tmpDir);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    // test stderr
    Assert.assertTrue(diagnostics.contains("error"));
    // test stdout
    Assert.assertTrue(shexc.getOutput().contains("hello"));
    Assert.assertTrue(shexc.getExitCode() == 2);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 
Example 27
Source Project: hadoop   Source File: BaseContainerManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
protected ContainerExecutor createContainerExecutor() {
  DefaultContainerExecutor exec = new DefaultContainerExecutor();
  exec.setConf(conf);
  return exec;
}
 
Example 28
Source Project: hadoop   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
        "apps_staging_dir/").getAbsolutePath());
  }

  // By default, VMEM monitoring disabled, PMEM monitoring enabled.
  if (!conf.getBoolean(
      MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
      MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
  }

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");

  try {
    Path stagingPath = FileContext.getFileContext(conf).makeQualified(
        new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
    /*
     * Re-configure the staging path on Windows if the file system is localFs.
     * We need to use a absolute path that contains the drive letter. The unit
     * test could run on a different drive than the AM. We can run into the
     * issue that job files are localized to the drive where the test runs on,
     * while the AM starts on a different drive and fails to find the job
     * metafiles. Using absolute path can avoid this ambiguity.
     */
    if (Path.WINDOWS) {
      if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
            new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
                .getAbsolutePath());
      }
    }
    FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
    if (fc.util().exists(stagingPath)) {
      LOG.info(stagingPath + " exists! deleting...");
      fc.delete(stagingPath, true);
    }
    LOG.info("mkdir: " + stagingPath);
    //mkdir the staging directory so that right permissions are set while running as proxy user
    fc.mkdir(stagingPath, null, true);
    //mkdir done directory as well 
    String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    Path doneDirPath = fc.makeQualified(new Path(doneDir));
    fc.mkdir(doneDirPath, null, true);
  } catch (IOException e) {
    throw new YarnRuntimeException("Could not create staging directory. ", e);
  }
  conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                           // which shuffle doesn't happen
  //configure the shuffle service in NM
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
      ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
      Service.class);

  // Non-standard shuffle port
  conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      DefaultContainerExecutor.class, ContainerExecutor.class);

  // TestMRJobs is for testing non-uberized operation only; see TestUberAM
  // for corresponding uberized tests.
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

  super.serviceInit(conf);
}
 
Example 29
Source Project: big-c   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
public ContainerManagerImpl(Context context, ContainerExecutor exec,
    DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
    NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
    LocalDirsHandlerService dirsHandler) {
  super(ContainerManagerImpl.class.getName());
  this.context = context;
  this.dirsHandler = dirsHandler;

  // ContainerManager level dispatcher.
  dispatcher = new AsyncDispatcher();
  this.deletionService = deletionContext;
  this.metrics = metrics;

  rsrcLocalizationSrvc =
      createResourceLocalizationService(exec, deletionContext, context);
  addService(rsrcLocalizationSrvc);

  containersLauncher = createContainersLauncher(context, exec);
  addService(containersLauncher);

  this.nodeStatusUpdater = nodeStatusUpdater;
  this.aclsManager = aclsManager;

  // Start configurable services
  auxiliaryServices = new AuxServices();
  auxiliaryServices.registerServiceListener(this);
  addService(auxiliaryServices);

  this.containersMonitor =
      new ContainersMonitorImpl(exec, dispatcher, this.context);
  addService(this.containersMonitor);

  dispatcher.register(ContainerEventType.class,
      new ContainerEventDispatcher());
  dispatcher.register(ApplicationEventType.class,
      new ApplicationEventDispatcher());
  dispatcher.register(LocalizationEventType.class, rsrcLocalizationSrvc);
  dispatcher.register(AuxServicesEventType.class, auxiliaryServices);
  dispatcher.register(ContainersMonitorEventType.class, containersMonitor);
  dispatcher.register(ContainersLauncherEventType.class, containersLauncher);
  
  addService(dispatcher);

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();
}
 
Example 30
Source Project: big-c   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
protected ResourceLocalizationService createResourceLocalizationService(
    ContainerExecutor exec, DeletionService deletionContext, Context context) {
  return new ResourceLocalizationService(this.dispatcher, exec,
      deletionContext, dirsHandler, context);
}