org.apache.hadoop.yarn.util.AuxiliaryServiceHelper Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.AuxiliaryServiceHelper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testAuxiliaryServiceHelper() throws Exception {
  Map<String, String> env = new HashMap<String, String>();
  String serviceName = "testAuxiliaryService";
  ByteBuffer bb = ByteBuffer.wrap("testAuxiliaryService".getBytes());
  AuxiliaryServiceHelper.setServiceDataIntoEnv(serviceName, bb, env);
  Assert.assertEquals(bb,
      AuxiliaryServiceHelper.getServiceDataFromEnv(serviceName, env));
}
 
Example #2
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testAuxiliaryServiceHelper() throws Exception {
  Map<String, String> env = new HashMap<String, String>();
  String serviceName = "testAuxiliaryService";
  ByteBuffer bb = ByteBuffer.wrap("testAuxiliaryService".getBytes());
  AuxiliaryServiceHelper.setServiceDataIntoEnv(serviceName, bb, env);
  Assert.assertEquals(bb,
      AuxiliaryServiceHelper.getServiceDataFromEnv(serviceName, env));
}
 
Example #3
Source File: TezTaskContextImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public ByteBuffer getServiceProviderMetaData(String serviceName) {
  Preconditions.checkNotNull(serviceName, "serviceName is null");
  return AuxiliaryServiceHelper.getServiceDataFromEnv(
      serviceName, auxServiceEnv);
}
 
Example #4
Source File: TezTaskContextImpl.java    From tez with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public ByteBuffer getServiceProviderMetaData(String serviceName) {
  Objects.requireNonNull(serviceName, "serviceName is null");
  return AuxiliaryServiceHelper.getServiceDataFromEnv(
      serviceName, auxServiceEnv);
}
 
Example #5
Source File: TestOnFileUnorderedKVOutput.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Test
public void testGeneratedDataMovementEvent() throws Exception {

  OnFileUnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest();

  Configuration conf = new Configuration();
  conf.set(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
  conf.set(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());

  int appAttemptNumber = 1;
  TezUmbilical tezUmbilical = null;
  String dagName = "currentDAG";
  String taskVertexName = "currentVertex";
  String destinationVertexName = "destinationVertex";
  TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
  TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
  TezCounters counters = new TezCounters();
  byte[] userPayload = TezUtils.createUserPayloadFromConf(conf);
  RuntimeTask runtimeTask = mock(RuntimeTask.class);
  
  int shufflePort = 2112;
  Map<String, String> auxEnv = new HashMap<String, String>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(shufflePort);
  bb.position(0);
  AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, bb, auxEnv);


  OutputDescriptor outputDescriptor = mock(OutputDescriptor.class);
  when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor");

  TezOutputContext outputContext = new TezOutputContextImpl(conf, new String[] {workDir.toString()},
      appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
      taskAttemptID, counters, 0, userPayload, runtimeTask,
      null, auxEnv, new MemoryDistributor(1, 1, conf) , outputDescriptor);

  List<Event> events = null;

  events = kvOutput.initialize(outputContext);
  assertTrue(events != null && events.size() == 0);

  KeyValueWriter kvWriter = kvOutput.getWriter();
  List<KVPair> data = KVDataGen.generateTestData(true);
  for (KVPair kvp : data) {
    kvWriter.write(kvp.getKey(), kvp.getvalue());
  }

  events = kvOutput.close();
  assertTrue(events != null && events.size() == 1);
  DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

  assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

  DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
      .parseFrom(dmEvent.getUserPayload());

  assertFalse(shufflePayload.hasEmptyPartitions());
  assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
  assertEquals(shufflePort, shufflePayload.getPort());
  assertEquals("host", shufflePayload.getHost());
}
 
Example #6
Source File: ContainerRunnerImpl.java    From tez with Apache License 2.0 4 votes vote down vote up
public void setShufflePort(String auxiliaryService, int shufflePort) {
  AuxiliaryServiceHelper.setServiceDataIntoEnv(
      auxiliaryService,
      ByteBuffer.allocate(4).putInt(shufflePort), localEnv);
}
 
Example #7
Source File: TestOnFileUnorderedKVOutput.java    From tez with Apache License 2.0 4 votes vote down vote up
private OutputContext createOutputContext(Configuration payloadConf, Configuration baseConf,
    TezSharedExecutor sharedExecutor) throws IOException {
  int appAttemptNumber = 1;
  TezUmbilical tezUmbilical = mock(TezUmbilical.class);
  String dagName = "currentDAG";
  String taskVertexName = "currentVertex";
  String destinationVertexName = "destinationVertex";
  TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
  TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
  UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
  
  TaskSpec mockSpec = mock(TaskSpec.class);
  when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class)));
  when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class)));
  task = new LogicalIOProcessorRuntimeTask(mockSpec, appAttemptNumber, new Configuration(),
      new String[]{"/"}, tezUmbilical, null, null, null, null, "", null, 1024, false,
      new DefaultHadoopShim(), sharedExecutor);

  LogicalIOProcessorRuntimeTask runtimeTask = spy(task);
  
  Map<String, String> auxEnv = new HashMap<String, String>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(shufflePort);
  bb.position(0);
  AuxiliaryServiceHelper.setServiceDataIntoEnv(payloadConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
      TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT), bb, auxEnv);


  OutputDescriptor outputDescriptor = mock(OutputDescriptor.class);
  when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor");

  OutputContext realOutputContext = new TezOutputContextImpl(baseConf, new String[] {workDir.toString()},
      appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
      -1, taskAttemptID, 0, userPayload, runtimeTask,
      null, auxEnv, new MemoryDistributor(1, 1, payloadConf), outputDescriptor, null,
      new ExecutionContextImpl("localhost"), 2048, new TezSharedExecutor(defaultConf));
  verify(runtimeTask, times(1)).addAndGetTezCounter(destinationVertexName);
  verify(runtimeTask, times(1)).getTaskStatistics();
  // verify output stats object got created
  Assert.assertTrue(task.getTaskStatistics().getIOStatistics().containsKey(destinationVertexName));
  OutputContext outputContext = spy(realOutputContext);
  doAnswer(new Answer() {
    @Override public Object answer(InvocationOnMock invocation) throws Throwable {
      long requestedSize = (Long) invocation.getArguments()[0];
      MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
          .getArguments()[1];
      callback.memoryAssigned(requestedSize);
      return null;
    }
  }).when(outputContext).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));

  return outputContext;
}
 
Example #8
Source File: MapUtils.java    From tez with Apache License 2.0 4 votes vote down vote up
public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Path workDir,
    JobConf jobConf, int mapId, Path mapInput,
    TezUmbilical umbilical, String dagName,
    String vertexName, List<InputSpec> inputSpecs,
    List<OutputSpec> outputSpecs, TezSharedExecutor sharedExecutor) throws Exception {
  jobConf.setInputFormat(SequenceFileInputFormat.class);

  ProcessorDescriptor mapProcessorDesc = ProcessorDescriptor.create(
      MapProcessor.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(jobConf));
  
  Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

  TaskSpec taskSpec = new TaskSpec(
      TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0),
      dagName, vertexName, -1,
      mapProcessorDesc,
      inputSpecs,
      outputSpecs, null, null);

  Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
  String auxiliaryService = jobConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
      TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
  serviceConsumerMetadata.put(auxiliaryService,
      ShuffleUtils.convertJobTokenToBytes(shuffleToken));
  Map<String, String> envMap = new HashMap<String, String>();
  ByteBuffer shufflePortBb = ByteBuffer.allocate(4).putInt(0, 8000);
  AuxiliaryServiceHelper
      .setServiceDataIntoEnv(auxiliaryService, shufflePortBb,
          envMap);

  LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(
      taskSpec,
      0,
      jobConf,
      new String[] {workDir.toString()},
      umbilical,
      serviceConsumerMetadata,
      envMap,
      HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"),
      Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
  return task;
}
 
Example #9
Source File: LocalContainerLauncher.java    From tez with Apache License 2.0 4 votes vote down vote up
public LocalContainerLauncher(ContainerLauncherContext containerLauncherContext,
                              AppContext context,
                              TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                              String workingDirectory,
                              boolean isLocalMode) throws UnknownHostException, TezException {
  // TODO Post TEZ-2003. Most of this information is dynamic and only available after the AM
  // starts up. It's not possible to set these up via a static payload.
  // Will need some kind of mechanism to dynamically crate payloads / bind to parameters
  // after the AM starts up.
  super(containerLauncherContext);
  this.context = context;
  this.tal = taskCommunicatorManagerInterface;
  this.workingDirectory = workingDirectory;
  this.isLocalMode = isLocalMode;

  // Check if the hostname is set in the environment before overriding it.
  String host = isLocalMode ? InetAddress.getLocalHost().getHostName() :
      System.getenv(Environment.NM_HOST.name());
  executionContext = new ExecutionContextImpl(host);

  Configuration conf;
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to parse user payload for " + LocalContainerLauncher.class.getSimpleName(), e);
  }
  if (isLocalMode) {
    String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
        TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
    localEnv = Maps.newHashMap();
    shufflePort = 0;
    AuxiliaryServiceHelper.setServiceDataIntoEnv(
        auxiliaryService, ByteBuffer.allocate(4).putInt(shufflePort), localEnv);
  } else {
    localEnv = System.getenv();
  }
  numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
      TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT);
  Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor");
  ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors,
      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LocalTaskExecutionThread #%d")
          .build());
  this.taskExecutorService = MoreExecutors.listeningDecorator(rawExecutor);
  boolean cleanupDagDataOnComplete = ShuffleUtils.isTezShuffleHandler(conf)
      && conf.getBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION,
      TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT);
  if (cleanupDagDataOnComplete) {
    String deletionTrackerClassName = conf.get(TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS,
        TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS_DEFAULT);
    deletionTracker = ReflectionUtils.createClazzInstance(
        deletionTrackerClassName, new Class[]{Configuration.class}, new Object[]{conf});
  }
}