org.apache.tez.common.TezUtils Java Examples

The following examples show how to use org.apache.tez.common.TezUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleTestDAG.java    From tez with Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 
Example #2
Source File: ShuffledUnorderedKVInput.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws Exception {
  Preconditions.checkArgument(getNumPhysicalInputs() != -1, "Number of Inputs has not been set");
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());

  if (getNumPhysicalInputs() == 0) {
    getContext().requestInitialMemory(0l, null);
    isStarted.set(true);
    getContext().inputIsReady();
    LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
        + getContext().getSourceVertexName());
    return Collections.emptyList();
  } else {
    long initalMemReq = getInitialMemoryReq();
    memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    this.getContext().requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler);
  }

  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.inputRecordCounter = getContext().getCounters().findCounter(
      TaskCounter.INPUT_RECORDS_PROCESSED);
  return Collections.emptyList();
}
 
Example #3
Source File: TezTestServiceContainerLauncher.java    From tez with Apache License 2.0 6 votes vote down vote up
public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  int numThreads = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);

  this.servicePort = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
  Preconditions.checkArgument(servicePort > 0,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
  this.communicator = new TezTestServiceCommunicator(numThreads);
  this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString();
  this.appAttemptId = getContext().getApplicationAttemptId();
}
 
Example #4
Source File: MockDAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up
@Override
protected ContainerLauncherManager createContainerLauncherManager(
    List<NamedEntityDescriptor> containerLauncherDescirptors,
    boolean isLocal)
    throws UnknownHostException {
  UserPayload userPayload;
  try {
    userPayload = TezUtils.createUserPayloadFromConf(new Configuration(false));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  ContainerLauncherManager clManager = new ContainerLauncherManager(getContext());
  ContainerLauncherContext containerLauncherContext =
      new ContainerLauncherContextImpl(getContext(), clManager, getTaskCommunicatorManager(), userPayload, 0);
  containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext);
  clManager.setContainerLauncher(containerLauncher);
  return clManager;
}
 
Example #5
Source File: VertexManager.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public void initialize() {
  pluginContext = new VertexManagerPluginContextImpl();
  if (pluginDesc != null) {
    plugin = ReflectionUtils.createClazzInstance(pluginDesc.getClassName());
    payload = DagTypeConverters.convertToTezUserPayload(pluginDesc.getUserPayload());
  }
  if (payload == null || payload.getPayload() == null) {
    // Ease of use. If no payload present then give the common configuration
    // TODO TEZ-744 Don't do this - AMConf should not be used to configure vertexManagers.
    try {
      payload = DagTypeConverters.convertToTezUserPayload(
          TezUtils.createUserPayloadFromConf(appContext.getAMConf()));
    } catch (IOException e) {
      throw new TezUncheckedException(e);
    }
  }
  plugin.initialize(pluginContext);
}
 
Example #6
Source File: OnFileSortedOutput.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws IOException {
  this.startTime = System.nanoTime();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  // Initializing this parametr in this conf since it is used in multiple
  // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
  // TezMerger, etc.
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      ExternalSorter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

  sendEmptyPartitionDetails = this.conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);
  return Collections.emptyList();
}
 
Example #7
Source File: OutputTestHelpers.java    From tez with Apache License 2.0 6 votes vote down vote up
static OutputContext createOutputContext(Configuration conf, Configuration userPayloadConf, Path workingDir)
    throws IOException {
  OutputContext ctx = mock(OutputContext.class);
  doAnswer(new Answer<Void>() {
    @Override public Void answer(InvocationOnMock invocation) throws Throwable {
      long requestedSize = (Long) invocation.getArguments()[0];
      MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
          .getArguments()[1];
      callback.memoryAssigned(requestedSize);
      return null;
    }
  }).when(ctx).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
  doReturn(conf).when(ctx).getContainerConfiguration();
  doReturn(TezUtils.createUserPayloadFromConf(userPayloadConf)).when(ctx).getUserPayload();
  doReturn("destinationVertex").when(ctx).getDestinationVertexName();
  doReturn("UUID").when(ctx).getUniqueIdentifier();
  doReturn(new String[] { workingDir.toString() }).when(ctx).getWorkDirs();
  doReturn(200 * 1024 * 1024l).when(ctx).getTotalMemoryAvailableToTask();
  doReturn(new TezCounters()).when(ctx).getCounters();
  OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class);
  doReturn(statsReporter).when(ctx).getStatisticsReporter();
  doReturn(new ExecutionContextImpl("localhost")).when(ctx).getExecutionContext();
  return ctx;
}
 
Example #8
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 6 votes vote down vote up
@Override
public List<Event> initialize()  throws Exception {
  MRInputUserPayloadProto userPayloadProto = MRInputHelpers
      .parseMRInputPayload(getContext().getInputUserPayload());
  Configuration conf = TezUtils.createConfFromByteString(userPayloadProto
      .getConfigurationBytes());

  try {
    Thread.currentThread().setContextClassLoader(TezClassLoader.getInstance());
    ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME);
    LOG.info("Class found");
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(new Path("/tmp/relocalizationfilefound"));
  } catch (TezReflectionException e) {
    LOG.info("Class not found");
  }

  return super.initialize();
}
 
Example #9
Source File: SimpleTestDAG3Vertices.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 
Example #10
Source File: OutputTestHelpers.java    From tez with Apache License 2.0 6 votes vote down vote up
static OutputContext createOutputContext() throws IOException {
  OutputContext outputContext = mock(OutputContext.class);
  Configuration conf = new TezConfiguration();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  String[] workingDirs = new String[]{"workDir1"};
  OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class);
  TezCounters counters = new TezCounters();

  doReturn("destinationVertex").when(outputContext).getDestinationVertexName();
  doReturn(payLoad).when(outputContext).getUserPayload();
  doReturn(workingDirs).when(outputContext).getWorkDirs();
  doReturn(200 * 1024 * 1024l).when(outputContext).getTotalMemoryAvailableToTask();
  doReturn(counters).when(outputContext).getCounters();
  doReturn(statsReporter).when(outputContext).getStatisticsReporter();
  doReturn(new Configuration(false)).when(outputContext).getContainerConfiguration();
  return outputContext;
}
 
Example #11
Source File: TestAMRecovery.java    From tez with Apache License 2.0 6 votes vote down vote up
/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
 
Example #12
Source File: SimpleTestDAG.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}
 
Example #13
Source File: TestOnFileSortedOutput.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 5000)
public void testPipelinedShuffleWithFinalMerge() throws Exception {
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name());

  //wrong setting for final merge enable in output
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
  OutputContext context = createTezOutputContext();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  doReturn(payLoad).when(context).getUserPayload();
  sortedOutput = new OrderedPartitionedKVOutput(context, partitions);

  sortedOutput.initialize();
  sortedOutput.start();
  assertFalse(sortedOutput.finalMergeEnabled); //should be disabled as pipelining is on
  assertTrue(sortedOutput.pipelinedShuffle);
}
 
Example #14
Source File: UnorderedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Get a UserPayload representation of the Configuration
 * @return a {@link org.apache.tez.dag.api.UserPayload} instance
 */
public UserPayload toUserPayload() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #15
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
String toHistoryText() {
  if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
      TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
    return TezUtils.convertToHistoryText(conf);
  }
  return null;
}
 
Example #16
Source File: TestOnFileSortedOutput.java    From tez with Apache License 2.0 5 votes vote down vote up
private void startSortedOutput(int partitions) throws Exception {
  OutputContext context = createTezOutputContext();
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 4);
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  doReturn(payLoad).when(context).getUserPayload();
  sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
  sortedOutput.initialize();
  sortedOutput.start();
  writer = sortedOutput.getWriter();
}
 
Example #17
Source File: FairShuffleVertexManager.java    From tez with Apache License 2.0 5 votes vote down vote up
public VertexManagerPluginDescriptor build() {
  VertexManagerPluginDescriptor desc =
      VertexManagerPluginDescriptor.create(
          FairShuffleVertexManager.class.getName());

  try {
    return desc.setUserPayload(TezUtils.createUserPayloadFromConf(
        this.conf));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
}
 
Example #18
Source File: ShuffleVertexManager.java    From tez with Apache License 2.0 5 votes vote down vote up
public VertexManagerPluginDescriptor build() {
  VertexManagerPluginDescriptor desc =
      VertexManagerPluginDescriptor.create(
          ShuffleVertexManager.class.getName());

  try {
    return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
}
 
Example #19
Source File: TestDAGAppMaster.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testParseAllPluginsOnlyCustomSpecified() throws IOException {
  Configuration conf = new Configuration(false);
  conf.set(TEST_KEY, TEST_VAL);
  UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
  TezUserPayloadProto payloadProto = TezUserPayloadProto.newBuilder()
      .setUserPayload(ByteString.copyFrom(defaultPayload.getPayload())).build();

  AMPluginDescriptorProto proto = createAmPluginDescriptor(false, false, true, payloadProto);

  List<NamedEntityDescriptor> tsDescriptors;
  BiMap<String, Integer> tsMap;
  List<NamedEntityDescriptor> clDescriptors;
  BiMap<String, Integer> clMap;
  List<NamedEntityDescriptor> tcDescriptors;
  BiMap<String, Integer> tcMap;


  // Only plugin, Yarn.
  tsDescriptors = Lists.newLinkedList();
  tsMap = HashBiMap.create();
  clDescriptors = Lists.newLinkedList();
  clMap = HashBiMap.create();
  tcDescriptors = Lists.newLinkedList();
  tcMap = HashBiMap.create();
  DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap,
      proto, false, defaultPayload);
  verifyDescAndMap(tsDescriptors, tsMap, 2, true, TS_NAME,
      TezConstants.getTezYarnServicePluginName());
  verifyDescAndMap(clDescriptors, clMap, 1, true, CL_NAME);
  verifyDescAndMap(tcDescriptors, tcMap, 1, true, TC_NAME);
  assertEquals(TS_NAME + CLASS_SUFFIX, tsDescriptors.get(0).getClassName());
  assertEquals(CL_NAME + CLASS_SUFFIX, clDescriptors.get(0).getClassName());
  assertEquals(TC_NAME + CLASS_SUFFIX, tcDescriptors.get(0).getClassName());
}
 
Example #20
Source File: MRCombiner.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public MRCombiner(TezTaskContext taskContext) throws IOException {
  this.conf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload());

  assert(taskContext instanceof TezInputContext || taskContext instanceof TezOutputContext);
  if (taskContext instanceof TezOutputContext) {
    this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezOutputContext)taskContext);
  } else {
    this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezInputContext)taskContext);
  }

  this.useNewApi = ConfigUtils.useNewApi(conf);
  
  combineInputKeyCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
  combineInputValueCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
  
  boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false);
  this.mrTaskAttemptID = new TaskAttemptID(
      new TaskID(String.valueOf(taskContext.getApplicationId()
          .getClusterTimestamp()), taskContext.getApplicationId().getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE,
          taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber());
  
  LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi);
}
 
Example #21
Source File: TestExternalTezServices.java    From tez with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {

  extServiceTestHelper = new ExternalTezServiceTestHelper(TEST_ROOT_DIR);
  UserPayload userPayload = TezUtils.createUserPayloadFromConf(extServiceTestHelper.getConfForJobs());

  TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[]{
      TaskSchedulerDescriptor
          .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName())
          .setUserPayload(userPayload)};

  ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[]{
      ContainerLauncherDescriptor
          .create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName())
          .setUserPayload(userPayload)};

  TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[]{
      TaskCommunicatorDescriptor
          .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName())
          .setUserPayload(userPayload)};

  ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true,
      taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);


  extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor);


  // Generate the join data set used for each run.
  // Can a timeout be enforced here ?
  Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
  Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
  extServiceTestHelper
      .setupHashJoinData(SRC_DATA_DIR, dataPath1, dataPath2, HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH);
}
 
Example #22
Source File: MROutput.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Creates the user payload to be set on the OutputDescriptor for MROutput
 * @param conf Configuration for the OutputFormat
 * @param outputFormatName Name of the class of the OutputFormat
 * @param useNewApi Use new mapreduce API or old mapred API
 * @return
 * @throws IOException
 */
public static byte[] createUserPayload(Configuration conf, 
    String outputFormatName, boolean useNewApi) throws IOException {
  Configuration outputConf = new JobConf(conf);
  outputConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, outputFormatName);
  outputConf.setBoolean("mapred.mapper.new-api", useNewApi);
  MRHelpers.translateVertexConfToTez(outputConf);
  MRHelpers.doJobClientMagic(outputConf);
  return TezUtils.createUserPayloadFromConf(outputConf);
}
 
Example #23
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Get a UserPayload representation of the Configuration
 * @return a {@link org.apache.tez.dag.api.UserPayload} instance
 */
public UserPayload toUserPayload() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #24
Source File: OrderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #25
Source File: TestDAGAppMaster.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testParseAllPluginsNoneSpecified() throws IOException {
  Configuration conf = new Configuration(false);
  conf.set(TEST_KEY, TEST_VAL);
  UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);

  List<NamedEntityDescriptor> tsDescriptors;
  BiMap<String, Integer> tsMap;
  List<NamedEntityDescriptor> clDescriptors;
  BiMap<String, Integer> clMap;
  List<NamedEntityDescriptor> tcDescriptors;
  BiMap<String, Integer> tcMap;


  // No plugins. Non local
  tsDescriptors = Lists.newLinkedList();
  tsMap = HashBiMap.create();
  clDescriptors = Lists.newLinkedList();
  clMap = HashBiMap.create();
  tcDescriptors = Lists.newLinkedList();
  tcMap = HashBiMap.create();
  DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap,
      null, false, defaultPayload);
  verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezYarnServicePluginName());
  verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezYarnServicePluginName());
  verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezYarnServicePluginName());

  // No plugins. Local
  tsDescriptors = Lists.newLinkedList();
  tsMap = HashBiMap.create();
  clDescriptors = Lists.newLinkedList();
  clMap = HashBiMap.create();
  tcDescriptors = Lists.newLinkedList();
  tcMap = HashBiMap.create();
  DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap,
      null, true, defaultPayload);
  verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezUberServicePluginName());
  verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezUberServicePluginName());
  verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezUberServicePluginName());
}
 
Example #26
Source File: TestShuffleVertexManager.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private ShuffleVertexManager createManager(Configuration conf, 
    VertexManagerPluginContext context, float min, float max) {
  conf.setFloat(ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min);
  conf.setFloat(ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max);    
  ShuffleVertexManager manager = new ShuffleVertexManager();
  byte[] payload;
  try {
    payload = TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  when(context.getUserPayload()).thenReturn(payload);
  manager.initialize(context);
  return manager;
}
 
Example #27
Source File: TestShuffleInputEventHandlerImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private ByteString createEmptyPartitionByteString(int... emptyPartitions) throws IOException {
  BitSet bitSet = new BitSet();
  for (int i : emptyPartitions) {
    bitSet.set(i);
  }
  ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtils
      .toByteArray(bitSet));
  return emptyPartitionsBytesString;
}
 
Example #28
Source File: TestOrderedGroupedKVInput.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test
public void testMergeConfig() throws IOException, TezException {
  Configuration baseConf = new Configuration(false);
  baseConf.set("base-key", "base-value");

  Configuration payloadConf = new Configuration(false);
  payloadConf.set("local-key", "local-value");

  InputContext inputContext = mock(InputContext.class);

  UserPayload payLoad = TezUtils.createUserPayloadFromConf(payloadConf);
  String[] workingDirs = new String[]{"workDir1"};
  TezCounters counters = new TezCounters();


  doReturn(payLoad).when(inputContext).getUserPayload();
  doReturn(workingDirs).when(inputContext).getWorkDirs();
  doReturn(counters).when(inputContext).getCounters();
  doReturn(baseConf).when(inputContext).getContainerConfiguration();

  OrderedGroupedKVInput input = new OrderedGroupedKVInput(inputContext, 1);
  input.initialize();

  Configuration mergedConf = input.conf;
  assertEquals("base-value", mergedConf.get("base-key"));
  assertEquals("local-value", mergedConf.get("local-key"));
}
 
Example #29
Source File: OnFileSortedOutputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Get a byte array representation of the configuration
 * @return a byte array which can be used as the payload
 */
public byte[] toByteArray() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #30
Source File: OnFileUnorderedPartitionedKVOutputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
public void fromByteArray(byte[] payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}