org.apache.tez.common.TezUtils Java Examples
The following examples show how to use
org.apache.tez.common.TezUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleTestDAG.java From tez with Apache License 2.0 | 6 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = DAG.create(name); Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #2
Source File: ShuffledUnorderedKVInput.java From incubator-tez with Apache License 2.0 | 6 votes |
@Override public synchronized List<Event> initialize() throws Exception { Preconditions.checkArgument(getNumPhysicalInputs() != -1, "Number of Inputs has not been set"); this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); if (getNumPhysicalInputs() == 0) { getContext().requestInitialMemory(0l, null); isStarted.set(true); getContext().inputIsReady(); LOG.info("input fetch not required since there are 0 physical inputs for input vertex: " + getContext().getSourceVertexName()); return Collections.emptyList(); } else { long initalMemReq = getInitialMemoryReq(); memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler(); this.getContext().requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler); } this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs()); this.inputRecordCounter = getContext().getCounters().findCounter( TaskCounter.INPUT_RECORDS_PROCESSED); return Collections.emptyList(); }
Example #3
Source File: TezTestServiceContainerLauncher.java From tez with Apache License 2.0 | 6 votes |
public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) { super(containerLauncherContext); try { conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload()); } catch (IOException e) { throw new RuntimeException(e); } int numThreads = conf.getInt( TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS, TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT); this.servicePort = conf.getInt( TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1); Preconditions.checkArgument(servicePort > 0, TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set"); this.communicator = new TezTestServiceCommunicator(numThreads); this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString(); this.appAttemptId = getContext().getApplicationAttemptId(); }
Example #4
Source File: MockDAGAppMaster.java From tez with Apache License 2.0 | 6 votes |
@Override protected ContainerLauncherManager createContainerLauncherManager( List<NamedEntityDescriptor> containerLauncherDescirptors, boolean isLocal) throws UnknownHostException { UserPayload userPayload; try { userPayload = TezUtils.createUserPayloadFromConf(new Configuration(false)); } catch (IOException e) { throw new TezUncheckedException(e); } ContainerLauncherManager clManager = new ContainerLauncherManager(getContext()); ContainerLauncherContext containerLauncherContext = new ContainerLauncherContextImpl(getContext(), clManager, getTaskCommunicatorManager(), userPayload, 0); containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext); clManager.setContainerLauncher(containerLauncher); return clManager; }
Example #5
Source File: VertexManager.java From incubator-tez with Apache License 2.0 | 6 votes |
public void initialize() { pluginContext = new VertexManagerPluginContextImpl(); if (pluginDesc != null) { plugin = ReflectionUtils.createClazzInstance(pluginDesc.getClassName()); payload = DagTypeConverters.convertToTezUserPayload(pluginDesc.getUserPayload()); } if (payload == null || payload.getPayload() == null) { // Ease of use. If no payload present then give the common configuration // TODO TEZ-744 Don't do this - AMConf should not be used to configure vertexManagers. try { payload = DagTypeConverters.convertToTezUserPayload( TezUtils.createUserPayloadFromConf(appContext.getAMConf())); } catch (IOException e) { throw new TezUncheckedException(e); } } plugin.initialize(pluginContext); }
Example #6
Source File: OnFileSortedOutput.java From incubator-tez with Apache License 2.0 | 6 votes |
@Override public synchronized List<Event> initialize() throws IOException { this.startTime = System.nanoTime(); this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); // Initializing this parametr in this conf since it is used in multiple // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles, // TezMerger, etc. this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs()); this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler(); getContext().requestInitialMemory( ExternalSorter.getInitialMemoryRequirement(conf, getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler); sendEmptyPartitionDetails = this.conf.getBoolean( TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED, TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT); return Collections.emptyList(); }
Example #7
Source File: OutputTestHelpers.java From tez with Apache License 2.0 | 6 votes |
static OutputContext createOutputContext(Configuration conf, Configuration userPayloadConf, Path workingDir) throws IOException { OutputContext ctx = mock(OutputContext.class); doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { long requestedSize = (Long) invocation.getArguments()[0]; MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation .getArguments()[1]; callback.memoryAssigned(requestedSize); return null; } }).when(ctx).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class)); doReturn(conf).when(ctx).getContainerConfiguration(); doReturn(TezUtils.createUserPayloadFromConf(userPayloadConf)).when(ctx).getUserPayload(); doReturn("destinationVertex").when(ctx).getDestinationVertexName(); doReturn("UUID").when(ctx).getUniqueIdentifier(); doReturn(new String[] { workingDir.toString() }).when(ctx).getWorkDirs(); doReturn(200 * 1024 * 1024l).when(ctx).getTotalMemoryAvailableToTask(); doReturn(new TezCounters()).when(ctx).getCounters(); OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class); doReturn(statsReporter).when(ctx).getStatisticsReporter(); doReturn(new ExecutionContextImpl("localhost")).when(ctx).getExecutionContext(); return ctx; }
Example #8
Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0 | 6 votes |
@Override public List<Event> initialize() throws Exception { MRInputUserPayloadProto userPayloadProto = MRInputHelpers .parseMRInputPayload(getContext().getInputUserPayload()); Configuration conf = TezUtils.createConfFromByteString(userPayloadProto .getConfigurationBytes()); try { Thread.currentThread().setContextClassLoader(TezClassLoader.getInstance()); ReflectionUtils.getClazz(RELOCALIZATION_TEST_CLASS_NAME); LOG.info("Class found"); FileSystem fs = FileSystem.get(conf); fs.mkdirs(new Path("/tmp/relocalizationfilefound")); } catch (TezReflectionException e) { LOG.info("Class not found"); } return super.initialize(); }
Example #9
Source File: SimpleTestDAG3Vertices.java From incubator-tez with Apache License 2.0 | 6 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { byte[] payload = null; int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = new DAG(name); Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource); dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); dag.addVertex(v3).addEdge(new Edge(v2, v3, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #10
Source File: OutputTestHelpers.java From tez with Apache License 2.0 | 6 votes |
static OutputContext createOutputContext() throws IOException { OutputContext outputContext = mock(OutputContext.class); Configuration conf = new TezConfiguration(); UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); String[] workingDirs = new String[]{"workDir1"}; OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class); TezCounters counters = new TezCounters(); doReturn("destinationVertex").when(outputContext).getDestinationVertexName(); doReturn(payLoad).when(outputContext).getUserPayload(); doReturn(workingDirs).when(outputContext).getWorkDirs(); doReturn(200 * 1024 * 1024l).when(outputContext).getTotalMemoryAvailableToTask(); doReturn(counters).when(outputContext).getCounters(); doReturn(statsReporter).when(outputContext).getStatisticsReporter(); doReturn(new Configuration(false)).when(outputContext).getContainerConfiguration(); return outputContext; }
Example #11
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 6 votes |
/** * v1 --> v2 <br> * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case. * v2 has a customized VM which could control when to kill AM * * @param vertexManagerClass * @param dmType * @param failOnParitialCompleted * @return * @throws IOException */ private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType, boolean failOnParitialCompleted) throws IOException { if (failOnParitialCompleted) { tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true"); } else { tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false"); } DAG dag = DAG.create(dagName); UserPayload payload = UserPayload.create(null); Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2); v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( ScheduleControlledVertexManager.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(tezConf))); Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2); v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( vertexManagerClass.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(tezConf))); dag.addVertex(v1).addVertex(v2); dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #12
Source File: SimpleTestDAG.java From incubator-tez with Apache License 2.0 | 6 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { byte[] payload = null; int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = new DAG(name); Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #13
Source File: TestOnFileSortedOutput.java From tez with Apache License 2.0 | 6 votes |
@Test (timeout = 5000) public void testPipelinedShuffleWithFinalMerge() throws Exception { conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name()); //wrong setting for final merge enable in output conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true); OutputContext context = createTezOutputContext(); UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); doReturn(payLoad).when(context).getUserPayload(); sortedOutput = new OrderedPartitionedKVOutput(context, partitions); sortedOutput.initialize(); sortedOutput.start(); assertFalse(sortedOutput.finalMergeEnabled); //should be disabled as pipelining is on assertTrue(sortedOutput.pipelinedShuffle); }
Example #14
Source File: UnorderedKVOutputConfig.java From tez with Apache License 2.0 | 5 votes |
/** * Get a UserPayload representation of the Configuration * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { try { return TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } }
Example #15
Source File: OrderedGroupedKVInputConfig.java From tez with Apache License 2.0 | 5 votes |
@InterfaceAudience.Private String toHistoryText() { if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { return TezUtils.convertToHistoryText(conf); } return null; }
Example #16
Source File: TestOnFileSortedOutput.java From tez with Apache License 2.0 | 5 votes |
private void startSortedOutput(int partitions) throws Exception { OutputContext context = createTezOutputContext(); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 4); UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); doReturn(payLoad).when(context).getUserPayload(); sortedOutput = new OrderedPartitionedKVOutput(context, partitions); sortedOutput.initialize(); sortedOutput.start(); writer = sortedOutput.getWriter(); }
Example #17
Source File: FairShuffleVertexManager.java From tez with Apache License 2.0 | 5 votes |
public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( FairShuffleVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils.createUserPayloadFromConf( this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } }
Example #18
Source File: ShuffleVertexManager.java From tez with Apache License 2.0 | 5 votes |
public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( ShuffleVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } }
Example #19
Source File: TestDAGAppMaster.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testParseAllPluginsOnlyCustomSpecified() throws IOException { Configuration conf = new Configuration(false); conf.set(TEST_KEY, TEST_VAL); UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf); TezUserPayloadProto payloadProto = TezUserPayloadProto.newBuilder() .setUserPayload(ByteString.copyFrom(defaultPayload.getPayload())).build(); AMPluginDescriptorProto proto = createAmPluginDescriptor(false, false, true, payloadProto); List<NamedEntityDescriptor> tsDescriptors; BiMap<String, Integer> tsMap; List<NamedEntityDescriptor> clDescriptors; BiMap<String, Integer> clMap; List<NamedEntityDescriptor> tcDescriptors; BiMap<String, Integer> tcMap; // Only plugin, Yarn. tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); clDescriptors = Lists.newLinkedList(); clMap = HashBiMap.create(); tcDescriptors = Lists.newLinkedList(); tcMap = HashBiMap.create(); DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, proto, false, defaultPayload); verifyDescAndMap(tsDescriptors, tsMap, 2, true, TS_NAME, TezConstants.getTezYarnServicePluginName()); verifyDescAndMap(clDescriptors, clMap, 1, true, CL_NAME); verifyDescAndMap(tcDescriptors, tcMap, 1, true, TC_NAME); assertEquals(TS_NAME + CLASS_SUFFIX, tsDescriptors.get(0).getClassName()); assertEquals(CL_NAME + CLASS_SUFFIX, clDescriptors.get(0).getClassName()); assertEquals(TC_NAME + CLASS_SUFFIX, tcDescriptors.get(0).getClassName()); }
Example #20
Source File: MRCombiner.java From incubator-tez with Apache License 2.0 | 5 votes |
public MRCombiner(TezTaskContext taskContext) throws IOException { this.conf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload()); assert(taskContext instanceof TezInputContext || taskContext instanceof TezOutputContext); if (taskContext instanceof TezOutputContext) { this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf); this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf); this.reporter = new MRTaskReporter((TezOutputContext)taskContext); } else { this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateInputValueClass(conf); this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf); this.reporter = new MRTaskReporter((TezInputContext)taskContext); } this.useNewApi = ConfigUtils.useNewApi(conf); combineInputKeyCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); combineInputValueCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false); this.mrTaskAttemptID = new TaskAttemptID( new TaskID(String.valueOf(taskContext.getApplicationId() .getClusterTimestamp()), taskContext.getApplicationId().getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber()); LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi); }
Example #21
Source File: TestExternalTezServices.java From tez with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { extServiceTestHelper = new ExternalTezServiceTestHelper(TEST_ROOT_DIR); UserPayload userPayload = TezUtils.createUserPayloadFromConf(extServiceTestHelper.getConfForJobs()); TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[]{ TaskSchedulerDescriptor .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName()) .setUserPayload(userPayload)}; ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[]{ ContainerLauncherDescriptor .create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName()) .setUserPayload(userPayload)}; TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[]{ TaskCommunicatorDescriptor .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName()) .setUserPayload(userPayload)}; ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors); extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor); // Generate the join data set used for each run. // Can a timeout be enforced here ? Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1"); Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2"); extServiceTestHelper .setupHashJoinData(SRC_DATA_DIR, dataPath1, dataPath2, HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH); }
Example #22
Source File: MROutput.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Creates the user payload to be set on the OutputDescriptor for MROutput * @param conf Configuration for the OutputFormat * @param outputFormatName Name of the class of the OutputFormat * @param useNewApi Use new mapreduce API or old mapred API * @return * @throws IOException */ public static byte[] createUserPayload(Configuration conf, String outputFormatName, boolean useNewApi) throws IOException { Configuration outputConf = new JobConf(conf); outputConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, outputFormatName); outputConf.setBoolean("mapred.mapper.new-api", useNewApi); MRHelpers.translateVertexConfToTez(outputConf); MRHelpers.doJobClientMagic(outputConf); return TezUtils.createUserPayloadFromConf(outputConf); }
Example #23
Source File: OrderedGroupedKVInputConfig.java From tez with Apache License 2.0 | 5 votes |
/** * Get a UserPayload representation of the Configuration * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { try { return TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } }
Example #24
Source File: OrderedPartitionedKVOutputConfig.java From tez with Apache License 2.0 | 5 votes |
@InterfaceAudience.Private public void fromUserPayload(UserPayload payload) { try { this.conf = TezUtils.createConfFromUserPayload(payload); } catch (IOException e) { throw new RuntimeException(e); } }
Example #25
Source File: TestDAGAppMaster.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testParseAllPluginsNoneSpecified() throws IOException { Configuration conf = new Configuration(false); conf.set(TEST_KEY, TEST_VAL); UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf); List<NamedEntityDescriptor> tsDescriptors; BiMap<String, Integer> tsMap; List<NamedEntityDescriptor> clDescriptors; BiMap<String, Integer> clMap; List<NamedEntityDescriptor> tcDescriptors; BiMap<String, Integer> tcMap; // No plugins. Non local tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); clDescriptors = Lists.newLinkedList(); clMap = HashBiMap.create(); tcDescriptors = Lists.newLinkedList(); tcMap = HashBiMap.create(); DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, null, false, defaultPayload); verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezYarnServicePluginName()); verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezYarnServicePluginName()); verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezYarnServicePluginName()); // No plugins. Local tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); clDescriptors = Lists.newLinkedList(); clMap = HashBiMap.create(); tcDescriptors = Lists.newLinkedList(); tcMap = HashBiMap.create(); DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, null, true, defaultPayload); verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezUberServicePluginName()); verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezUberServicePluginName()); verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezUberServicePluginName()); }
Example #26
Source File: TestShuffleVertexManager.java From incubator-tez with Apache License 2.0 | 5 votes |
private ShuffleVertexManager createManager(Configuration conf, VertexManagerPluginContext context, float min, float max) { conf.setFloat(ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min); conf.setFloat(ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max); ShuffleVertexManager manager = new ShuffleVertexManager(); byte[] payload; try { payload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } when(context.getUserPayload()).thenReturn(payload); manager.initialize(context); return manager; }
Example #27
Source File: TestShuffleInputEventHandlerImpl.java From incubator-tez with Apache License 2.0 | 5 votes |
private ByteString createEmptyPartitionByteString(int... emptyPartitions) throws IOException { BitSet bitSet = new BitSet(); for (int i : emptyPartitions) { bitSet.set(i); } ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtils .toByteArray(bitSet)); return emptyPartitionsBytesString; }
Example #28
Source File: TestOrderedGroupedKVInput.java From tez with Apache License 2.0 | 5 votes |
@Test public void testMergeConfig() throws IOException, TezException { Configuration baseConf = new Configuration(false); baseConf.set("base-key", "base-value"); Configuration payloadConf = new Configuration(false); payloadConf.set("local-key", "local-value"); InputContext inputContext = mock(InputContext.class); UserPayload payLoad = TezUtils.createUserPayloadFromConf(payloadConf); String[] workingDirs = new String[]{"workDir1"}; TezCounters counters = new TezCounters(); doReturn(payLoad).when(inputContext).getUserPayload(); doReturn(workingDirs).when(inputContext).getWorkDirs(); doReturn(counters).when(inputContext).getCounters(); doReturn(baseConf).when(inputContext).getContainerConfiguration(); OrderedGroupedKVInput input = new OrderedGroupedKVInput(inputContext, 1); input.initialize(); Configuration mergedConf = input.conf; assertEquals("base-value", mergedConf.get("base-key")); assertEquals("local-value", mergedConf.get("local-key")); }
Example #29
Source File: OnFileSortedOutputConfiguration.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Get a byte array representation of the configuration * @return a byte array which can be used as the payload */ public byte[] toByteArray() { try { return TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } }
Example #30
Source File: OnFileUnorderedPartitionedKVOutputConfiguration.java From incubator-tez with Apache License 2.0 | 5 votes |
@InterfaceAudience.Private public void fromByteArray(byte[] payload) { try { this.conf = TezUtils.createConfFromUserPayload(payload); } catch (IOException e) { throw new RuntimeException(e); } }