Java Code Examples for org.apache.tez.common.TezUtils

The following examples show how to use org.apache.tez.common.TezUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-tez   Source File: SimpleTestDAG.java    License: Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}
 
Example 2
Source Project: tez   Source File: TestAMRecovery.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
 
Example 3
Source Project: tez   Source File: OutputTestHelpers.java    License: Apache License 2.0 6 votes vote down vote up
static OutputContext createOutputContext() throws IOException {
  OutputContext outputContext = mock(OutputContext.class);
  Configuration conf = new TezConfiguration();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  String[] workingDirs = new String[]{"workDir1"};
  OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class);
  TezCounters counters = new TezCounters();

  doReturn("destinationVertex").when(outputContext).getDestinationVertexName();
  doReturn(payLoad).when(outputContext).getUserPayload();
  doReturn(workingDirs).when(outputContext).getWorkDirs();
  doReturn(200 * 1024 * 1024l).when(outputContext).getTotalMemoryAvailableToTask();
  doReturn(counters).when(outputContext).getCounters();
  doReturn(statsReporter).when(outputContext).getStatisticsReporter();
  doReturn(new Configuration(false)).when(outputContext).getContainerConfiguration();
  return outputContext;
}
 
Example 4
Source Project: incubator-tez   Source File: SimpleTestDAG3Vertices.java    License: Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 
Example 5
Source Project: tez   Source File: OutputTestHelpers.java    License: Apache License 2.0 6 votes vote down vote up
static OutputContext createOutputContext(Configuration conf, Configuration userPayloadConf, Path workingDir)
    throws IOException {
  OutputContext ctx = mock(OutputContext.class);
  doAnswer(new Answer<Void>() {
    @Override public Void answer(InvocationOnMock invocation) throws Throwable {
      long requestedSize = (Long) invocation.getArguments()[0];
      MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
          .getArguments()[1];
      callback.memoryAssigned(requestedSize);
      return null;
    }
  }).when(ctx).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
  doReturn(conf).when(ctx).getContainerConfiguration();
  doReturn(TezUtils.createUserPayloadFromConf(userPayloadConf)).when(ctx).getUserPayload();
  doReturn("destinationVertex").when(ctx).getDestinationVertexName();
  doReturn("UUID").when(ctx).getUniqueIdentifier();
  doReturn(new String[] { workingDir.toString() }).when(ctx).getWorkDirs();
  doReturn(200 * 1024 * 1024l).when(ctx).getTotalMemoryAvailableToTask();
  doReturn(new TezCounters()).when(ctx).getCounters();
  OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class);
  doReturn(statsReporter).when(ctx).getStatisticsReporter();
  doReturn(new ExecutionContextImpl("localhost")).when(ctx).getExecutionContext();
  return ctx;
}
 
Example 6
Source Project: incubator-tez   Source File: OnFileSortedOutput.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws IOException {
  this.startTime = System.nanoTime();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  // Initializing this parametr in this conf since it is used in multiple
  // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
  // TezMerger, etc.
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      ExternalSorter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

  sendEmptyPartitionDetails = this.conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);
  return Collections.emptyList();
}
 
Example 7
Source Project: incubator-tez   Source File: ShuffledUnorderedKVInput.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws Exception {
  Preconditions.checkArgument(getNumPhysicalInputs() != -1, "Number of Inputs has not been set");
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());

  if (getNumPhysicalInputs() == 0) {
    getContext().requestInitialMemory(0l, null);
    isStarted.set(true);
    getContext().inputIsReady();
    LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
        + getContext().getSourceVertexName());
    return Collections.emptyList();
  } else {
    long initalMemReq = getInitialMemoryReq();
    memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    this.getContext().requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler);
  }

  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.inputRecordCounter = getContext().getCounters().findCounter(
      TaskCounter.INPUT_RECORDS_PROCESSED);
  return Collections.emptyList();
}
 
Example 8
Source Project: tez   Source File: TestOnFileSortedOutput.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 5000)
public void testPipelinedShuffleWithFinalMerge() throws Exception {
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name());

  //wrong setting for final merge enable in output
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
  OutputContext context = createTezOutputContext();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  doReturn(payLoad).when(context).getUserPayload();
  sortedOutput = new OrderedPartitionedKVOutput(context, partitions);

  sortedOutput.initialize();
  sortedOutput.start();
  assertFalse(sortedOutput.finalMergeEnabled); //should be disabled as pipelining is on
  assertTrue(sortedOutput.pipelinedShuffle);
}
 
Example 9
Source Project: tez   Source File: MockDAGAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ContainerLauncherManager createContainerLauncherManager(
    List<NamedEntityDescriptor> containerLauncherDescirptors,
    boolean isLocal)
    throws UnknownHostException {
  UserPayload userPayload;
  try {
    userPayload = TezUtils.createUserPayloadFromConf(new Configuration(false));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  ContainerLauncherManager clManager = new ContainerLauncherManager(getContext());
  ContainerLauncherContext containerLauncherContext =
      new ContainerLauncherContextImpl(getContext(), clManager, getTaskCommunicatorManager(), userPayload, 0);
  containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext);
  clManager.setContainerLauncher(containerLauncher);
  return clManager;
}
 
Example 10
Source Project: incubator-tez   Source File: VertexManager.java    License: Apache License 2.0 6 votes vote down vote up
public void initialize() {
  pluginContext = new VertexManagerPluginContextImpl();
  if (pluginDesc != null) {
    plugin = ReflectionUtils.createClazzInstance(pluginDesc.getClassName());
    payload = DagTypeConverters.convertToTezUserPayload(pluginDesc.getUserPayload());
  }
  if (payload == null || payload.getPayload() == null) {
    // Ease of use. If no payload present then give the common configuration
    // TODO TEZ-744 Don't do this - AMConf should not be used to configure vertexManagers.
    try {
      payload = DagTypeConverters.convertToTezUserPayload(
          TezUtils.createUserPayloadFromConf(appContext.getAMConf()));
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }
  }
  plugin.initialize(pluginContext);
}
 
Example 11
Source Project: tez   Source File: TezTestServiceContainerLauncher.java    License: Apache License 2.0 6 votes vote down vote up
public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  int numThreads = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);

  this.servicePort = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
  Preconditions.checkArgument(servicePort > 0,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
  this.communicator = new TezTestServiceCommunicator(numThreads);
  this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString();
  this.appAttemptId = getContext().getApplicationAttemptId();
}
 
Example 12
Source Project: tez   Source File: TestMRRJobsDAGApi.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<Event> initialize()  throws Exception {
  MRInputUserPayloadProto userPayloadProto = MRInputHelpers
      .parseMRInputPayload(getContext().getInputUserPayload());
  Configuration conf = TezUtils.createConfFromByteString(userPayloadProto
      .getConfigurationBytes());

  try {
    Thread.currentThread().setContextClassLoader(TezClassLoader.getInstance());
    ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME);
    LOG.info("Class found");
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(new Path("/tmp/relocalizationfilefound"));
  } catch (TezReflectionException e) {
    LOG.info("Class not found");
  }

  return super.initialize();
}
 
Example 13
Source Project: tez   Source File: SimpleTestDAG.java    License: Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 
Example 14
Source Project: tez   Source File: TestExceptionPropagation.java    License: Apache License 2.0 5 votes vote down vote up
public static VertexManagerPluginDescriptor getVMDesc(ExceptionLocation exLocation) throws IOException {
  Configuration conf = new Configuration();
  conf.set(Test_ExceptionLocation, exLocation.name());
  UserPayload payload = TezUtils.createUserPayloadFromConf(conf);
  return VertexManagerPluginDescriptor.create(InputReadyVertexManagerWithException.class.getName())
          .setUserPayload(payload);
}
 
Example 15
Source Project: tez   Source File: ShuffleVertexManagerBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  config = initConfiguration();
  updatePendingTasks();
  if (config.isAutoParallelismEnabled()) {
    getContext().vertexReconfigurationPlanned();
  }
  // dont track the source tasks here since those tasks may themselves be
  // dynamically changed as the DAG progresses.
}
 
Example 16
Source Project: tez   Source File: TestEntityDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyResults(InputDescriptor entityDescriptor, InputDescriptor deserialized, UserPayload payload,
                           String confVal) throws IOException {
  Assert.assertEquals(entityDescriptor.getClassName(), deserialized.getClassName());
  // History text is not serialized when sending to tasks
  Assert.assertNull(deserialized.getHistoryText());
  Assert.assertArrayEquals(payload.deepCopyAsArray(), deserialized.getUserPayload().deepCopyAsArray());
  Configuration deserializedConf = TezUtils.createConfFromUserPayload(deserialized.getUserPayload());
  Assert.assertEquals(confVal, deserializedConf.get("testKey"));
}
 
Example 17
Source Project: incubator-tez   Source File: TestProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(TezProcessorContext processorContext) throws Exception {
  this.processorContext = processorContext;
  if (processorContext.getUserPayload() != null) {
    String vName = processorContext.getTaskVertexName();
    conf = TezUtils.createConfFromUserPayload(processorContext
        .getUserPayload());
    verifyValue = conf.getInt(
        getVertexConfName(TEZ_FAILING_PROCESSOR_VERIFY_VALUE, vName,
            processorContext.getTaskIndex()), -1);
    if (verifyValue != -1) {
      LOG.info("Verify value: " + verifyValue);
      for (String verifyIndex : conf
          .getTrimmedStringCollection(
              getVertexConfName(TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, vName))) {
        LOG.info("Adding verify task index: " + verifyIndex);
        verifyTaskIndices.add(Integer.valueOf(verifyIndex));
      }
    }
    doFail = conf.getBoolean(
        getVertexConfName(TEZ_FAILING_PROCESSOR_DO_FAIL, vName), false);
    sleepMs = conf.getLong(
        getVertexConfName(TEZ_FAILING_PROCESSOR_SLEEP_MS, vName), 0);
    LOG.info("doFail: " + doFail);
    if (doFail) {
      for (String failingIndex : conf
          .getTrimmedStringCollection(
              getVertexConfName(TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, vName))) {
        LOG.info("Adding failing task index: " + failingIndex);
        failingTaskIndices.add(Integer.valueOf(failingIndex));
      }
      failingTaskAttemptUpto = conf.getInt(
          getVertexConfName(TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, vName), 0);
      LOG.info("Adding failing attempt : " + failingTaskAttemptUpto + 
          " dag: " + processorContext.getDAGName());
    }
  }
}
 
Example 18
Source Project: incubator-tez   Source File: SimpleReverseVTestDAG.java    License: Apache License 2.0 5 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addEdge(new Edge(v1, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 
Example 19
Source Project: incubator-tez   Source File: TwoLevelsFailingDAG.java    License: Apache License 2.0 5 votes vote down vote up
public static DAG createDAG(String name, 
        Configuration conf) throws Exception {
    if (conf != null) {
      payload = TezUtils.createUserPayloadFromConf(conf);
    } 
    dag = new DAG(name);
    addDAGVerticesAndEdges();
    return dag;
}
 
Example 20
Source Project: incubator-tez   Source File: SixLevelsFailingDAG.java    License: Apache License 2.0 5 votes vote down vote up
public static DAG createDAG(String name, 
        Configuration conf) throws Exception {
    if (conf != null) {
      payload = TezUtils.createUserPayloadFromConf(conf);
    } 
    dag = new DAG(name);
    addDAGVerticesAndEdges();
    return dag;
}
 
Example 21
Source Project: incubator-tez   Source File: ThreeLevelsFailingDAG.java    License: Apache License 2.0 5 votes vote down vote up
public static DAG createDAG(String name, 
        Configuration conf) throws Exception {
    if (conf != null) {
      payload = TezUtils.createUserPayloadFromConf(conf);
    } 
    dag = new DAG(name);
    addDAGVerticesAndEdges();
    return dag;
}
 
Example 22
Source Project: tez   Source File: OrderedGroupedKVInput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws IOException {
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());

  if (this.getNumPhysicalInputs() == 0) {
    getContext().requestInitialMemory(0l, null);
    isStarted.set(true);
    getContext().inputIsReady();
    LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
        + getContext().getSourceVertexName());
    return Collections.emptyList();
  }

  long initialMemoryRequest = Shuffle.getInitialMemoryRequirement(conf,
      getContext().getTotalMemoryAvailableToTask());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(initialMemoryRequest, memoryUpdateCallbackHandler);

  this.inputKeyCounter = getContext().getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
  this.inputValueCounter = getContext().getCounters().findCounter(
      TaskCounter.REDUCE_INPUT_RECORDS);
   this.shuffledInputs = getContext().getCounters().findCounter(
      TaskCounter.NUM_SHUFFLED_INPUTS);
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());

  return Collections.emptyList();
}
 
Example 23
Source Project: tez   Source File: TestDefaultSorter.java    License: Apache License 2.0 5 votes vote down vote up
private OutputContext createTezOutputContext() throws IOException {
  String[] workingDirs = { workingDir.toString() };
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
  serviceProviderMetaData.writeInt(PORT);

  TezCounters counters = new TezCounters();

  OutputContext context = mock(OutputContext.class);
  ExecutionContext execContext = new ExecutionContextImpl("localhost");
  doReturn(mock(OutputStatisticsReporter.class)).when(context).getStatisticsReporter();
  doReturn(execContext).when(context).getExecutionContext();
  doReturn(counters).when(context).getCounters();
  doReturn(workingDirs).when(context).getWorkDirs();
  doReturn(payLoad).when(context).getUserPayload();
  doReturn(5 * 1024 * 1024l).when(context).getTotalMemoryAvailableToTask();
  doReturn(UniqueID).when(context).getUniqueIdentifier();
  doReturn("v1").when(context).getDestinationVertexName();
  doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(context)
      .getServiceProviderMetaData
          (conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
              TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
  doAnswer(new Answer() {
    @Override public Object answer(InvocationOnMock invocation) throws Throwable {
      long requestedSize = (Long) invocation.getArguments()[0];
      MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
          .getArguments()[1];
      callback.memoryAssigned(requestedSize);
      return null;
    }
  }).when(context).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
  return context;
}
 
Example 24
Source Project: incubator-tez   Source File: LocalMergedInput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<Event> initialize() throws IOException {
  getContext().requestInitialMemory(0l, null); // mandatory call.
  getContext().inputIsReady();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());

  if (getNumPhysicalInputs() == 0) {
    return Collections.emptyList();
  }

  LocalShuffle localShuffle = new LocalShuffle(getContext(), conf, getNumPhysicalInputs());
  rawIter = localShuffle.run();
  createValuesIterator();
  return Collections.emptyList();
}
 
Example 25
Source Project: tez   Source File: GraceShuffleVertexManagerForTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  try {
    Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    graceConf = GraceConf.fromConfiguration(conf);
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  getContext().registerForVertexStateUpdates(graceConf.grandparentVertex,
      EnumSet.of(VertexState.SUCCEEDED));
  logger.info("Watching {}", graceConf.grandparentVertex);
  super.initialize();
}
 
Example 26
Source Project: tez   Source File: TestAMRecovery.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 
Example 27
Source Project: tez   Source File: UnorderedKVOutputConfig.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 28
private Event generateEvent() throws IOException {
  DataMovementEventPayloadProto.Builder payloadBuidler = DataMovementEventPayloadProto
      .newBuilder();

  String host = getHost();
  int shufflePort = getShufflePort();

  BitSet emptyPartitions = new BitSet();
  for (int i = 0; i < numPartitions; i++) {
    if (numRecordsPerPartition[i] == 0) {
      emptyPartitions.set(i);
    }
  }
  if (emptyPartitions.cardinality() != 0) {
    // Empty partitions exist
    ByteString emptyPartitionsByteString = TezCommonUtils.compressByteArrayToByteString(TezUtils
        .toByteArray(emptyPartitions));
    payloadBuidler.setEmptyPartitions(emptyPartitionsByteString);
  }
  if (emptyPartitions.cardinality() != numPartitions) {
    // Populate payload only if at least 1 partition has data
    payloadBuidler.setHost(host);
    payloadBuidler.setPort(shufflePort);
    payloadBuidler.setPathComponent(outputContext.getUniqueIdentifier());
  }

  CompositeDataMovementEvent cDme = new CompositeDataMovementEvent(0, numPartitions,
      payloadBuidler.build().toByteArray());
  return cDme;
}
 
Example 29
/**
 * Get a byte array representation of the configuration
 * @return a byte array which can be used as the payload
 */
public byte[] toByteArray() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 30
/**
 * Get a byte array representation of the configuration
 * @return a byte array which can be used as the payload
 */
public byte[] toByteArray() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}