Java Code Examples for org.apache.hadoop.io.MapWritable#entrySet()

The following examples show how to use org.apache.hadoop.io.MapWritable#entrySet() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestWritableUtil.java From datawave with Apache License 2.0

5 votes

/**
 * Reads a map of Strings from a DataInput.
 *
 * @param input
 * @return map of strings
 * @throws IOException
 */
public static Map<String,String> readMap(DataInput input) throws IOException {
    MapWritable mw = new MapWritable();
    mw.readFields(input);
    
    Map<String,String> map = new TreeMap<>();
    
    for (Map.Entry<Writable,Writable> entry : mw.entrySet()) {
        map.put(entry.getKey().toString(), entry.getValue().toString());
    }
    
    return map;
}

Example 2

Source File: TypedBytesWritableOutput.java From hadoop with Apache License 2.0

5 votes

public void writeMap(MapWritable mw) throws IOException {
  out.writeMapHeader(mw.size());
  for (Map.Entry<Writable, Writable> entry : mw.entrySet()) {
    write(entry.getKey());
    write(entry.getValue());
  }
}

Example 3

Source File: SolrHiveWriter.java From hive-solr with MIT License

5 votes

@Override
    public void write(Writable w) throws IOException {
        MapWritable map = (MapWritable) w;
        SolrInputDocument doc = new SolrInputDocument();
        for (final Map.Entry<Writable, Writable> entry : map.entrySet()) {
            String key = entry.getKey().toString();//得到key
            String value=entry.getValue().toString().trim();// null值会转成空字符串 得到value
            //只有value有值的数据，我们才推送到solr里面，无值数据，不再发送到solr里面
            if(value.length()>0){
            doc.setField(key,value);
            }
        }

//        count.incrementAndGet();


        datas.add(doc);
        //批量处理，大于等于一定量提交
        if(datas.size()==batchSize){
            try {
                sc.add(datas);
//                sc.commit();//不提交，等待flush
            }catch (Exception e){
                e.printStackTrace();
            }finally {
                //清空集合数据
                datas.clear();
            }
        }

    }

Example 4

Source File: TypedBytesWritableOutput.java From big-c with Apache License 2.0

5 votes

public void writeMap(MapWritable mw) throws IOException {
  out.writeMapHeader(mw.size());
  for (Map.Entry<Writable, Writable> entry : mw.entrySet()) {
    write(entry.getKey());
    write(entry.getValue());
  }
}

Example 5

Source File: XmlDataValidationMapper.java From jumbune with GNU Lesser General Public License v3.0

5 votes

private void cleanOutput(MapWritable mapErrorType) {

				Set<Map.Entry<Writable, Writable>> errorTypeEntrySet = mapErrorType.entrySet();

				for (Map.Entry<Writable, Writable> errorTypeMap : errorTypeEntrySet) {
					ArrayListWritable<XMLErrorWritable> errorList = (ArrayListWritable<XMLErrorWritable>) errorTypeMap.getValue();

					for (XMLErrorWritable error : errorList) {
						error = null;
					}
					errorList.clear();
				}
				errorTypeEntrySet.clear();
	}

Example 6

Source File: JsonMapReduce.java From hiped2 with Apache License 2.0

5 votes

@Override
protected void map(LongWritable key, MapWritable value,
                   Context context)
    throws
    IOException, InterruptedException {

  for (java.util.Map.Entry<Writable, Writable> entry : value
      .entrySet()) {
    context.write((Text) entry.getKey(), (Text) entry.getValue());
  }
}

Example 7

Source File: Wordcount.java From logparser with Apache License 2.0

5 votes

@Override
public void map(Object key, MapWritable value, Context context)
    throws IOException, InterruptedException {
    for (Map.Entry<Writable, Writable> entry : value.entrySet()) {
        word.set(entry.getValue().toString());
        context.write(word, ONE);
    }
}

Example 8

Source File: DataValidationMapper.java From jumbune with GNU Lesser General Public License v3.0

4 votes

private void writeViolations(DataDiscrepanciesArrayWritable dataValidationDiscripancies,
		DataViolationWB dataViolationWB, Context context, String outputKey)
				throws IOException, InterruptedException {		
	Text fName = new Text();
	fName.set(fileName);
	LongWritable startOffsetWritable = new LongWritable();
	LongWritable endOffsetWritable = new LongWritable();
	LongWritable recordsEmitMapWritable = new LongWritable();
	
	startOffsetWritable.set(splitStartOffset);
	endOffsetWritable.set(splitEndOffset);
	recordsEmitMapWritable.set(recordsEmittByMap);

	dataViolationWB.setSplitEndOffset(endOffsetWritable);
	dataViolationWB.setFileName(fName);
	dataViolationWB.setTotalRecordsEmittByMap(recordsEmitMapWritable);

	dataViolationWBArr[0] = dataViolationWB;
	dataValidationDiscripancies.set(dataViolationWBArr);
	StringBuilder builder = new StringBuilder();
	builder.append(outputKey).append(DDAW).append(dataValidationDiscripancies.get().hashCode());
	Text finalMapKey = new Text();
	finalMapKey.set(builder.toString());
	if(!dataViolationWB.getFieldMap().isEmpty()){
	context.write(finalMapKey, dataValidationDiscripancies);
	}
	//Clean up operation below
	MapWritable fieldMap = dataViolationWB.getFieldMap();

	for (Map.Entry<Writable, Writable> entry : fieldMap.entrySet()) {
		FieldLWB fieldLWB = (FieldLWB) fieldMap.get(entry.getKey());
		MapWritable typeViolateMap = fieldLWB.getTypeViolationMap();
		for (Map.Entry<Writable, Writable> tvEntry : typeViolateMap.entrySet()) {
			ViolationLWB vLWB = (ViolationLWB) typeViolateMap.get(tvEntry.getKey());
			vLWB.getLineLWBList().clear();
		}
		fieldLWB.getTypeViolationMap().clear();
		fieldLWB.resetTypeViolationMap();
	}		
	dataViolationWB.getFieldMap().clear();
	fieldMap = null;
	dataViolationWB.resetFieldMap();
	dataValidationDiscripancies = new DataDiscrepanciesArrayWritable();
	dataViolationWB = new DataViolationWB();
	
	if (outputKey.equals(DataValidationConstants.NUM_OF_FIELDS_CHECK)){
		lineVWCounterNOF = 0;	
	}else if(outputKey.equals(DataValidationConstants.USER_DEFINED_NULL_CHECK)) {
		lineVWCounterNC = 0;	
	}else if(outputKey.equals(DataValidationConstants.USER_DEFINED_DATA_TYPE)) {
		lineVWCounterDT = 0;	
	}else if (outputKey.equals(DataValidationConstants.USER_DEFINED_REGEX_CHECK)){
		lineVWCounterRX = 0;	
	}		
}