Java Code Examples for org.apache.hadoop.util.StringInterner#weakIntern()
The following examples show how to use
org.apache.hadoop.util.StringInterner#weakIntern() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TaskReport.java From big-c with Apache License 2.0 | 6 votes |
public void readFields(DataInput in) throws IOException { this.taskid.readFields(in); this.progress = in.readFloat(); this.state = StringInterner.weakIntern(Text.readString(in)); this.startTime = in.readLong(); this.finishTime = in.readLong(); diagnostics = WritableUtils.readStringArray(in); counters = new Counters(); counters.readFields(in); currentStatus = WritableUtils.readEnum(in, TIPStatus.class); if (currentStatus == TIPStatus.RUNNING) { int num = WritableUtils.readVInt(in); for (int i = 0; i < num; i++) { TaskAttemptID t = new TaskAttemptID(); t.readFields(in); runningAttempts.add(t); } } else if (currentStatus == TIPStatus.COMPLETE) { successfulAttempt.readFields(in); } }
Example 2
Source File: TezYARNUtils.java From incubator-tez with Apache License 2.0 | 6 votes |
public static String getFrameworkClasspath(Configuration conf) { Map<String, String> environment = new HashMap<String, String>(); TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(), Environment.PWD.$(), File.pathSeparator); TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", File.pathSeparator); // Add YARN/COMMON/HDFS jars and conf locations to path for (String c : conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(), c.trim(), File.pathSeparator); } return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name())); }
Example 3
Source File: TaskSpec.java From tez with Apache License 2.0 | 6 votes |
public TaskSpec( String dagName, String vertexName, int vertexParallelism, ProcessorDescriptor processorDescriptor, List<InputSpec> inputSpecList, List<OutputSpec> outputSpecList, @Nullable List<GroupInputSpec> groupInputSpecList, Configuration taskConf) { Objects.requireNonNull(dagName, "dagName is null"); Objects.requireNonNull(vertexName, "vertexName is null"); Objects.requireNonNull(processorDescriptor, "processorDescriptor is null"); Objects.requireNonNull(inputSpecList, "inputSpecList is null"); Objects.requireNonNull(outputSpecList, "outputSpecList is null"); this.taskAttemptId = null; this.dagName = StringInterner.weakIntern(dagName); this.vertexName = StringInterner.weakIntern(vertexName); this.processorDescriptor = processorDescriptor; this.inputSpecList = inputSpecList; this.outputSpecList = outputSpecList; this.groupInputSpecList = groupInputSpecList; this.vertexParallelism = vertexParallelism; this.taskConf = taskConf; }
Example 4
Source File: TaskAttemptListenerImpl.java From big-c with Apache License 2.0 | 6 votes |
@Override public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo) throws IOException { diagnosticInfo = StringInterner.weakIntern(diagnosticInfo); LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": " + diagnosticInfo); org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); taskHeartbeatHandler.progressing(attemptID); // This is mainly used for cases where we want to propagate exception traces // of tasks that fail. // This call exists as a hadoop mapreduce legacy wherein all changes in // counters/progress/phase/output-size are reported through statusUpdate() // call but not diagnosticInformation. context.getEventHandler().handle( new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo)); }
Example 5
Source File: JobHistoryParser.java From hadoop with Apache License 2.0 | 5 votes |
private void handleTaskFailedEvent(TaskFailedEvent event) { TaskInfo taskInfo = info.tasksMap.get(event.getTaskId()); taskInfo.status = TaskStatus.State.FAILED.toString(); taskInfo.finishTime = event.getFinishTime(); taskInfo.error = StringInterner.weakIntern(event.getError()); taskInfo.failedDueToAttemptId = event.getFailedAttemptID(); taskInfo.counters = event.getCounters(); }
Example 6
Source File: JobHistoryParser.java From hadoop with Apache License 2.0 | 5 votes |
private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) { TaskInfo taskInfo = info.tasksMap.get(event.getTaskId()); TaskAttemptInfo attemptInfo = taskInfo.attemptsMap.get(event.getAttemptId()); attemptInfo.finishTime = event.getFinishTime(); attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.counters = event.getCounters(); attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); }
Example 7
Source File: TaskSpec.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in); dagName = StringInterner.weakIntern(in.readUTF()); vertexName = StringInterner.weakIntern(in.readUTF()); // TODO TEZ-305 convert this to PB processorDescriptor = new ProcessorDescriptor(); processorDescriptor.readFields(in); int numInputSpecs = in.readInt(); inputSpecList = new ArrayList<InputSpec>(numInputSpecs); for (int i = 0; i < numInputSpecs; i++) { InputSpec inputSpec = new InputSpec(); inputSpec.readFields(in); inputSpecList.add(inputSpec); } int numOutputSpecs = in.readInt(); outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs); for (int i = 0; i < numOutputSpecs; i++) { OutputSpec outputSpec = new OutputSpec(); outputSpec.readFields(in); outputSpecList.add(outputSpec); } boolean hasGroupInputs = in.readBoolean(); if (hasGroupInputs) { int numGroups = in.readInt(); groupInputSpecList = Lists.newArrayListWithCapacity(numGroups); for (int i=0; i<numGroups; ++i) { GroupInputSpec group = new GroupInputSpec(); group.readFields(in); groupInputSpecList.add(group); } } }
Example 8
Source File: TaskSpec.java From tez with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in); dagName = StringInterner.weakIntern(in.readUTF()); vertexName = StringInterner.weakIntern(in.readUTF()); vertexParallelism = in.readInt(); // TODO TEZ-305 convert this to PB processorDescriptor = new ProcessorDescriptor(); processorDescriptor.readFields(in); int numInputSpecs = in.readInt(); inputSpecList = new ArrayList<InputSpec>(numInputSpecs); for (int i = 0; i < numInputSpecs; i++) { InputSpec inputSpec = new InputSpec(); inputSpec.readFields(in); inputSpecList.add(inputSpec); } int numOutputSpecs = in.readInt(); outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs); for (int i = 0; i < numOutputSpecs; i++) { OutputSpec outputSpec = new OutputSpec(); outputSpec.readFields(in); outputSpecList.add(outputSpec); } boolean hasGroupInputs = in.readBoolean(); if (hasGroupInputs) { int numGroups = in.readInt(); groupInputSpecList = Lists.newArrayListWithCapacity(numGroups); for (int i=0; i<numGroups; ++i) { GroupInputSpec group = new GroupInputSpec(); group.readFields(in); groupInputSpecList.add(group); } } boolean hasVertexConf = in.readBoolean(); if (hasVertexConf) { taskConf = new Configuration(false); taskConf.readFields(in); } }
Example 9
Source File: JobHistoryParser.java From big-c with Apache License 2.0 | 5 votes |
private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) { TaskInfo taskInfo = info.tasksMap.get(event.getTaskId()); TaskAttemptInfo attemptInfo = taskInfo.attemptsMap.get(event.getAttemptId()); attemptInfo.finishTime = event.getFinishTime(); attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.counters = event.getCounters(); attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); }
Example 10
Source File: Task.java From big-c with Apache License 2.0 | 5 votes |
public void readFields(DataInput in) throws IOException { jobFile = StringInterner.weakIntern(Text.readString(in)); taskId = TaskAttemptID.read(in); partition = in.readInt(); numSlotsRequired = in.readInt(); taskStatus.readFields(in); skipRanges.readFields(in); currentRecIndexIterator = skipRanges.skipRangeIterator(); currentRecStartIndex = currentRecIndexIterator.next(); skipping = in.readBoolean(); jobCleanup = in.readBoolean(); if (jobCleanup) { jobRunStateForCleanup = WritableUtils.readEnum(in, JobStatus.State.class); } jobSetup = in.readBoolean(); writeSkipRecs = in.readBoolean(); taskCleanup = in.readBoolean(); if (taskCleanup) { setPhase(TaskStatus.Phase.CLEANUP); } user = StringInterner.weakIntern(Text.readString(in)); int len = in.readInt(); encryptedSpillKey = new byte[len]; extraData.readFields(in); in.readFully(encryptedSpillKey); }
Example 11
Source File: ClusterStatus.java From hadoop with Apache License 2.0 | 5 votes |
public void readFields(DataInput in) throws IOException { numActiveTrackers = in.readInt(); int numTrackerNames = in.readInt(); if (numTrackerNames > 0) { for (int i = 0; i < numTrackerNames; i++) { String name = StringInterner.weakIntern(Text.readString(in)); activeTrackers.add(name); } } numBlacklistedTrackers = in.readInt(); int blackListTrackerInfoSize = in.readInt(); if(blackListTrackerInfoSize > 0) { for (int i = 0; i < blackListTrackerInfoSize; i++) { BlackListInfo info = new BlackListInfo(); info.readFields(in); blacklistedTrackersInfo.add(info); } } numExcludedNodes = in.readInt(); ttExpiryInterval = in.readLong(); map_tasks = in.readInt(); reduce_tasks = in.readInt(); max_map_tasks = in.readInt(); max_reduce_tasks = in.readInt(); status = WritableUtils.readEnum(in, JobTrackerStatus.class); grayListedTrackers = in.readInt(); }
Example 12
Source File: TaggedInputSplit.java From hadoop with Apache License 2.0 | 5 votes |
private Class<?> readClass(DataInput in) throws IOException { String className = StringInterner.weakIntern(Text.readString(in)); try { return conf.getClassByName(className); } catch (ClassNotFoundException e) { throw new RuntimeException("readObject can't find class", e); } }
Example 13
Source File: JobHistoryParser.java From hadoop with Apache License 2.0 | 5 votes |
private void handleAMStartedEvent(AMStartedEvent event) { AMInfo amInfo = new AMInfo(); amInfo.appAttemptId = event.getAppAttemptId(); amInfo.startTime = event.getStartTime(); amInfo.containerId = event.getContainerId(); amInfo.nodeManagerHost = StringInterner.weakIntern(event.getNodeManagerHost()); amInfo.nodeManagerPort = event.getNodeManagerPort(); amInfo.nodeManagerHttpPort = event.getNodeManagerHttpPort(); if (info.amInfos == null) { info.amInfos = new LinkedList<AMInfo>(); } info.amInfos.add(amInfo); info.latestAmInfo = amInfo; }
Example 14
Source File: JobHistoryParser.java From big-c with Apache License 2.0 | 5 votes |
private void handleTaskAttemptStartedEvent(TaskAttemptStartedEvent event) { TaskAttemptID attemptId = event.getTaskAttemptId(); TaskInfo taskInfo = info.tasksMap.get(event.getTaskId()); TaskAttemptInfo attemptInfo = new TaskAttemptInfo(); attemptInfo.startTime = event.getStartTime(); attemptInfo.attemptId = event.getTaskAttemptId(); attemptInfo.httpPort = event.getHttpPort(); attemptInfo.trackerName = StringInterner.weakIntern(event.getTrackerName()); attemptInfo.taskType = event.getTaskType(); attemptInfo.shufflePort = event.getShufflePort(); attemptInfo.containerId = event.getContainerId(); taskInfo.attemptsMap.put(attemptId, attemptInfo); }
Example 15
Source File: TaggedInputSplit.java From big-c with Apache License 2.0 | 5 votes |
private Class<?> readClass(DataInput in) throws IOException { String className = StringInterner.weakIntern(Text.readString(in)); try { return conf.getClassByName(className); } catch (ClassNotFoundException e) { throw new RuntimeException("readObject can't find class", e); } }
Example 16
Source File: TaskAttemptImpl.java From hadoop with Apache License 2.0 | 5 votes |
@Override public String getAssignedContainerMgrAddress() { readLock.lock(); try { return container == null ? null : StringInterner.weakIntern(container .getNodeId().toString()); } finally { readLock.unlock(); } }
Example 17
Source File: AbstractCounterGroup.java From tez with Apache License 2.0 | 4 votes |
public AbstractCounterGroup(String name, String displayName, Limits limits) { this.name = StringInterner.weakIntern(name); this.displayName = StringInterner.weakIntern(displayName); this.limits = limits; }
Example 18
Source File: DagInfo.java From tez with Apache License 2.0 | 4 votes |
DagInfo(JSONObject jsonObject) throws JSONException { super(jsonObject); vertexNameMap = Maps.newHashMap(); vertexNameIDMapping = new DualHashBidiMap<>(); edgeInfoMap = Maps.newHashMap(); basicVertexInfoMap = Maps.newHashMap(); containerMapping = LinkedHashMultimap.create(); Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_DAG_ID)); dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY)); //Parse additional Info JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); long sTime = otherInfoNode.optLong(Constants.START_TIME); long eTime= otherInfoNode.optLong(Constants.FINISH_TIME); userName = otherInfoNode.optString(Constants.USER); if (eTime < sTime) { LOG.warn("DAG has got wrong start/end values. " + "startTime=" + sTime + ", endTime=" + eTime + ". Will check " + "timestamps in DAG started/finished events"); // Check if events DAG_STARTED, DAG_FINISHED can be made use of for(Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { case DAG_STARTED: sTime = event.getAbsoluteTime(); break; case DAG_FINISHED: eTime = event.getAbsoluteTime(); break; default: break; } } if (eTime < sTime) { LOG.warn("DAG has got wrong start/end values in events as well. " + "startTime=" + sTime + ", endTime=" + eTime); } } startTime = sTime; endTime = eTime; //TODO: Not getting populated correctly for lots of jobs. Verify submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME); diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS); failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS); JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN); name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null); if (dagPlan != null) { JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES); if (vertices != null) { numVertices = vertices.length(); } else { numVertices = 0; } parseDAGPlan(dagPlan); } else { numVertices = 0; } status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS)); //parse name id mapping JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING); if (vertexIDMappingJson != null) { //get vertex name for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) { String vertexId = vertexIDMappingJson.optString(entry.getKey()); //vertexName --> vertexId vertexNameIDMapping.put(entry.getKey(), vertexId); } } }
Example 19
Source File: VertexImpl.java From incubator-tez with Apache License 2.0 | 4 votes |
public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName, Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, Clock clock, TaskHeartbeatHandler thh, boolean commitVertexOutputs, AppContext appContext, VertexLocationHint vertexLocationHint, Map<String, VertexGroupInfo> dagVertexGroups, JavaProfilerOptions profilerOpts) { this.vertexId = vertexId; this.vertexPlan = vertexPlan; this.vertexName = StringInterner.weakIntern(vertexName); this.conf = conf; this.clock = clock; this.appContext = appContext; this.commitVertexOutputs = commitVertexOutputs; this.taskAttemptListener = taskAttemptListener; this.taskHeartbeatHandler = thh; this.eventHandler = eventHandler; ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); if (LOG.isDebugEnabled()) { logLocationHints(this.vertexName, vertexLocationHint); } setTaskLocationHints(vertexLocationHint); this.dagUgi = appContext.getCurrentDAG().getDagUGI(); this.taskResource = DagTypeConverters .createResourceRequestFromTaskConfig(vertexPlan.getTaskConfig()); this.processorDescriptor = DagTypeConverters .convertProcessorDescriptorFromDAGPlan(vertexPlan .getProcessorDescriptor()); this.localResources = DagTypeConverters .createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig() .getLocalResourceList()); this.localResources.putAll(appContext.getSessionResources()); this.environment = DagTypeConverters .createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig() .getEnvironmentSettingList()); this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan .getTaskConfig().getJavaOpts() : null; this.profilerOpts = profilerOpts; this.containerContext = new ContainerContext(this.localResources, appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this); if (vertexPlan.getInputsCount() > 0) { setAdditionalInputs(vertexPlan.getInputsList()); } if (vertexPlan.getOutputsCount() > 0) { setAdditionalOutputs(vertexPlan.getOutputsList()); } // Setup the initial parallelism early. This may be changed after // initialization or on a setParallelism call. this.numTasks = vertexPlan.getTaskConfig().getNumTasks(); this.dagVertexGroups = dagVertexGroups; logIdentifier = this.getVertexId() + " [" + this.getName() + "]"; // This "this leak" is okay because the retained pointer is in an // instance variable. stateMachine = stateMachineFactory.make(this); }
Example 20
Source File: TaskInfo.java From tez with Apache License 2.0 | 4 votes |
TaskInfo(JSONObject jsonObject) throws JSONException { super(jsonObject); Preconditions.checkArgument( jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_TASK_ID)); taskId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY)); //Parse additional Info final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); long sTime = otherInfoNode.optLong(Constants.START_TIME); long eTime = otherInfoNode.optLong(Constants.FINISH_TIME); if (eTime < sTime) { LOG.warn("Task has got wrong start/end values. " + "startTime=" + sTime + ", endTime=" + eTime + ". Will check " + "timestamps in DAG started/finished events"); // Check if events TASK_STARTED, TASK_FINISHED can be made use of for(Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { case TASK_STARTED: sTime = event.getAbsoluteTime(); break; case TASK_FINISHED: eTime = event.getAbsoluteTime(); break; default: break; } } if (eTime < sTime) { LOG.warn("Task has got wrong start/end values in events as well. " + "startTime=" + sTime + ", endTime=" + eTime); } } startTime = sTime; endTime = eTime; diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS); successfulAttemptId = StringInterner.weakIntern( otherInfoNode.optString(Constants.SUCCESSFUL_ATTEMPT_ID)); scheduledTime = otherInfoNode.optLong(Constants.SCHEDULED_TIME); status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS)); }