Java Code Examples for org.apache.hadoop.io.Text#readFields()

public void readFields(DataInput dataInput) throws IOException {
    Text text = new Text();
    text.readFields(dataInput);

    Logger.println("value wrapper read class:"+text.toString());
    String className = text.toString();

    if (className.equals(IntWritable.class.getSimpleName())) {
        value = new IntWritable();
    } else if (className.equals(NewOldCustomElement.class.getSimpleName())) {
        value = new NewOldCustomElement();
    } else if (className.equals(CustomerFlowElement.class.getSimpleName())) {
        value = new CustomerFlowElement();
    } else {
       throw new IOException("can not read fields "+className);
    }
    value.readFields(dataInput);
}
 

@Override
  public void readFields(DataInput dataInput) throws IOException {
    Text formattedValueText =  (Text) WritableFactories.newInstance(Text.class);
    formattedValueText.readFields(dataInput);
    this.formattedValue=formattedValueText.toString();
    Text commentText = (Text) WritableFactories.newInstance(Text.class);
    commentText.readFields(dataInput);
    this.comment=commentText.toString();
    Text formulaText = (Text) WritableFactories.newInstance(Text.class);
    formulaText.readFields(dataInput);
    this.formula=formulaText.toString();
    Text addressText = (Text) WritableFactories.newInstance(Text.class);
    addressText.readFields(dataInput);
    this.address=addressText.toString();
    Text sheetNameText = (Text) WritableFactories.newInstance(Text.class);
    sheetNameText.readFields(dataInput);
    this.sheetName=sheetNameText.toString();
}
 

private void readOnDiskMapOutput(Configuration conf, FileSystem fs, Path path,
    List<String> keys, List<String> values) throws IOException {
  FSDataInputStream in = CryptoUtils.wrapIfNecessary(conf, fs.open(path));

  IFile.Reader<Text, Text> reader = new IFile.Reader<Text, Text>(conf, in,
      fs.getFileStatus(path).getLen(), null, null);
  DataInputBuffer keyBuff = new DataInputBuffer();
  DataInputBuffer valueBuff = new DataInputBuffer();
  Text key = new Text();
  Text value = new Text();
  while (reader.nextRawKey(keyBuff)) {
    key.readFields(keyBuff);
    keys.add(key.toString());
    reader.nextRawValue(valueBuff);
    value.readFields(valueBuff);
    values.add(value.toString());
  }
}
 

@Test
public void testSerialize()
    throws IOException {

  // Use our serializer, verify Hadoop deserializer can read it back
  for (String textToSerialize : textsToSerialize) {
    ByteArrayOutputStream bOs = new ByteArrayOutputStream();
    DataOutputStream dataOutputStream = new DataOutputStream(bOs);

    TextSerializer.writeStringAsText(dataOutputStream, textToSerialize);
    dataOutputStream.close();

    ByteArrayInputStream bIn = new ByteArrayInputStream(bOs.toByteArray());
    DataInputStream dataInputStream = new DataInputStream(bIn);

    Text hadoopText = new Text();
    hadoopText.readFields(dataInputStream);

    Assert.assertEquals(hadoopText.toString(), textToSerialize);
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
    start = in.readLong();
    length = in.readLong();
    Text forestIdText = new Text();
    forestIdText.readFields(in);
    forestId = new BigInteger(forestIdText.getBytes());
    hostName = new String[1];
    hostName[0] = Text.readString(in);
    isLastSplit = in.readBoolean();
    int replicaSize = in.readInt();
    replicas = new ArrayList<ForestHost>();
    for (int i=0; i < replicaSize; i++) {
        String curForest = Text.readString(in);
        String curHost = Text.readString(in);
        ForestHost fh = new ForestHost(curForest, curHost);
        replicas.add(fh);
    } 
}
 

@Override
public void readFields(DataInput in) throws IOException {
  Text outputNameText = new Text();
  outputNameText.readFields(in);
  outputNamePrefix = outputNameText.toString();
  outputNum = in.readInt();
  BooleanWritable hasSharedOutputs = new BooleanWritable();
  hasSharedOutputs.readFields(in);
  if (hasSharedOutputs.get()) {
    Text sharedOutputNamePrefixText = new Text();
    sharedOutputNamePrefixText.readFields(in);
    sharedOutputNamePrefix = sharedOutputNamePrefixText.toString();
    sharedOutputNum = in.readInt();
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  int sz = in.readInt();
  if (sz < 0) throw new IOException("Invalid size: " + sz + " for file metadata object");
  this.theMetadata = new TreeMap<Text, Text>();
  for (int i = 0; i < sz; i++) {
    Text key = new Text();
    Text val = new Text();
    key.readFields(in);
    val.readFields(in);
    this.theMetadata.put(key, val);
  }    
}
 

@Override
public void readFields(DataInput dataInput) throws IOException {
	this.values = new ArrayList<Text>();
	for (int i = 0; i < this.values.size(); ++i) {
		Text value = new Text();
		value.readFields(dataInput);
		this.values.set(i, value);
	}

}
 

public void readFields(DataInput in) throws IOException {
  id = new org.apache.hadoop.mapreduce.JobID();
  id.readFields(in);
  user = new Text();
  user.readFields(in);
  jobSubmitDir = new Path(WritableUtils.readString(in));
}
 

@Override
public void readFields(DataInput in) throws IOException {
  int sz = in.readInt();
  if (sz < 0) throw new IOException("Invalid size: " + sz + " for file metadata object");
  this.theMetadata = new TreeMap<Text, Text>();
  for (int i = 0; i < sz; i++) {
    Text key = new Text();
    Text val = new Text();
    key.readFields(in);
    val.readFields(in);
    this.theMetadata.put(key, val);
  }    
}
 

@Override
public void readFields(DataInput dataInput) throws IOException {
    this.typeId = dataInput.readByte();
    Text inputKey = new Text();
    inputKey.readFields(dataInput);
    init(inputKey, typeId);
}
 

public String[] getSorted() throws IOException {
  String[] ret = new String[indices.length];
  Text t = new Text();
  DataInputBuffer dib = new DataInputBuffer();
  for (int i = 0; i < ret.length; ++i) {
    int ii = indices[i];
    dib.reset(bytes, offsets[ii],
    ((ii + 1 == indices.length) ? eob : offsets[ii + 1]) - offsets[ii]);
    t.readFields(dib);
    ret[i] = t.toString();
  }
  return ret;
}
 

public void readFields(DataInput dataInput) throws IOException {
    Text text = new Text();
    text.readFields(dataInput);
    wifiProb = text.toString();

    IntWritable intReader = new IntWritable();

    intReader.readFields(dataInput);
    inNoOutWifi = intReader.get();
    intReader.readFields(dataInput);
    inNoOutStore = intReader.get();

    intReader.readFields(dataInput);
    outNoInWifi = intReader.get();
    intReader.readFields(dataInput);
    outNoInStore = intReader.get();


    intReader.readFields(dataInput);
    inAndOutWifi = intReader.get();
    intReader.readFields(dataInput);
    inAndOutStore = intReader.get();

    intReader.readFields(dataInput);
    stayInWifi = intReader.get();
    intReader.readFields(dataInput);
    stayInStore = intReader.get();


    DoubleWritable doubleWritable = new DoubleWritable();
    doubleWritable.readFields(dataInput);
    jumpRate = doubleWritable.get();
    doubleWritable.readFields(dataInput);
    deepVisit = doubleWritable.get();
    doubleWritable.readFields(dataInput);
    inStoreRate = doubleWritable.get();

}
 

@Override
public void readFields(DataInput in) throws IOException {
  Text text = new Text();
  text.readFields(in);
  this.jobId = text.toString().intern();
  text.readFields(in);
  this.taskId = text.toString().intern();
  this.taskAttemptId = Optional.absent();
  this.setId(this.taskId);
  this.startTime = in.readLong();
  this.endTime = in.readLong();
  this.duration = in.readLong();
  super.readFields(in);
}
 

@Override
public void readFields(DataInput paramDataInput)
    throws IOException
  {
    IntWritable localIntWritable = new IntWritable();

    localIntWritable.readFields(paramDataInput);
    this.int1 = localIntWritable.get();

    Text localText = new Text();
    localText.readFields(paramDataInput);
    this.circle = localText.toString();
  }
 

public int compare(byte[] b1, int s1, int l1,
                   byte[] b2, int s2, int l2) {
  
  if (sortSpec == null) {
    return super.compare(b1, s1, l1, b2, s2, l2);
  }
  
  try {
    Text logline1 = new Text(); 
    logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
    String line1 = logline1.toString();
    String[] logColumns1 = line1.split(columnSeparator);
    
    Text logline2 = new Text(); 
    logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
    String line2 = logline2.toString();
    String[] logColumns2 = line2.split(columnSeparator);
    
    if (logColumns1 == null || logColumns2 == null) {
      return super.compare(b1, s1, l1, b2, s2, l2);
    }
    
    //Compare column-wise according to *sortSpec*
    for(int i=0; i < sortSpec.length; ++i) {
      int column = (Integer.valueOf(sortSpec[i]).intValue());
      String c1 = logColumns1[column]; 
      String c2 = logColumns2[column];
      
      //Compare columns
      int comparision = super.compareBytes(
                                           c1.getBytes(), 0, c1.length(),
                                           c2.getBytes(), 0, c2.length()
                                           );
      
      //They differ!
      if (comparision != 0) {
        return comparision;
      }
    }
    
  } catch (IOException ioe) {
    LOG.fatal("Caught " + ioe);
    return 0;
  }
  
  return 0;
}
 

public int compare(byte[] b1, int s1, int l1,
                   byte[] b2, int s2, int l2) {
  
  if (sortSpec == null) {
    return super.compare(b1, s1, l1, b2, s2, l2);
  }
  
  try {
    Text logline1 = new Text(); 
    logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
    String line1 = logline1.toString();
    String[] logColumns1 = line1.split(columnSeparator);
    
    Text logline2 = new Text(); 
    logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
    String line2 = logline2.toString();
    String[] logColumns2 = line2.split(columnSeparator);
    
    if (logColumns1 == null || logColumns2 == null) {
      return super.compare(b1, s1, l1, b2, s2, l2);
    }
    
    //Compare column-wise according to *sortSpec*
    for(int i=0; i < sortSpec.length; ++i) {
      int column = (Integer.valueOf(sortSpec[i]).intValue());
      String c1 = logColumns1[column]; 
      String c2 = logColumns2[column];
      
      //Compare columns
      int comparision = super.compareBytes(
                                           c1.getBytes(), 0, c1.length(),
                                           c2.getBytes(), 0, c2.length()
                                           );
      
      //They differ!
      if (comparision != 0) {
        return comparision;
      }
    }
    
  } catch (IOException ioe) {
    LOG.fatal("Caught " + ioe);
    return 0;
  }
  
  return 0;
}
 

public int compare(byte[] b1, int s1, int l1,
                   byte[] b2, int s2, int l2) {
  
  if (sortSpec == null) {
    return super.compare(b1, s1, l1, b2, s2, l2);
  }
  
  try {
    Text logline1 = new Text(); 
    logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
    String line1 = logline1.toString();
    String[] logColumns1 = line1.split(columnSeparator);
    
    Text logline2 = new Text(); 
    logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
    String line2 = logline2.toString();
    String[] logColumns2 = line2.split(columnSeparator);
    
    if (logColumns1 == null || logColumns2 == null) {
      return super.compare(b1, s1, l1, b2, s2, l2);
    }
    
    //Compare column-wise according to *sortSpec*
    for(int i=0; i < sortSpec.length; ++i) {
      int column = Integer.parseInt(sortSpec[i]);
      String c1 = logColumns1[column]; 
      String c2 = logColumns2[column];
      
      //Compare columns
      int comparision = super.compareBytes(
                              c1.getBytes(Charset.forName("UTF-8")), 0, c1.length(),
                              c2.getBytes(Charset.forName("UTF-8")), 0, c2.length()
                              );
      
      //They differ!
      if (comparision != 0) {
        return comparision;
      }
    }
    
  } catch (IOException ioe) {
    LOG.fatal("Caught " + ioe);
    return 0;
  }
  
  return 0;
}
 

@Test(timeout = 5000)
  public void testMapProcessor() throws Exception {
    String dagName = "mrdag0";
    String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
    JobConf jobConf = new JobConf(defaultConf);
    setUpJobConf(jobConf);

    MRHelpers.translateMRConfToTez(jobConf);
    jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0);

    jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);

    jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
        "localized-resources").toUri().toString());
    
    Path mapInput = new Path(workDir, "map0");
    
    
    MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10);

    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        InputDescriptor.create(MRInputLegacy.class.getName())
            .setUserPayload(UserPayload.create(ByteBuffer.wrap(
                MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
                    .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build()
                    .toByteArray()))),
        1);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", 
        OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);

    TezSharedExecutor sharedExecutor = new TezSharedExecutor(jobConf);
    LogicalIOProcessorRuntimeTask task = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0,
        new Path(workDir, "map0"), new TestUmbilical(), dagName, vertexName,
        Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec),
        sharedExecutor);

    task.initialize();
    task.run();
    task.close();
    sharedExecutor.shutdownNow();

    OutputContext outputContext = task.getOutputContexts().iterator().next();
    TezTaskOutput mapOutputs = new TezTaskOutputFiles(
        jobConf, outputContext.getUniqueIdentifier(),
        outputContext.getDagIdentifier());
    
    
    // TODO NEWTEZ FIXME OutputCommitter verification
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
//        .getCommitter().getClass().getName());
//    t.close();

    Path mapOutputFile = getMapOutputFile(jobConf, outputContext);
    LOG.info("mapOutputFile = " + mapOutputFile);
    IFile.Reader reader =
        new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1);
    LongWritable key = new LongWritable();
    Text value = new Text();
    DataInputBuffer keyBuf = new DataInputBuffer();
    DataInputBuffer valueBuf = new DataInputBuffer();
    long prev = Long.MIN_VALUE;
    while (reader.nextRawKey(keyBuf)) {
      reader.nextRawValue(valueBuf);
      key.readFields(keyBuf);
      value.readFields(valueBuf);
      if (prev != Long.MIN_VALUE) {
        assert(prev <= key.get());
        prev = key.get();
      }
      LOG.info("key = " + key.get() + "; value = " + value);
    }
    reader.close();
  }
 

@Override
public void readFields(DataInput in) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.readFields(in);
    tms = tms_text.toString();

    // 1. integers
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.readFields(in);
        num[i] = intw.get();
    }

    intw.readFields(in);
    int1 = intw.get();

    intw.readFields(in);
    int2 = intw.get();

    // 2. strings
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.readFields(in);
        strings[i] = txt.toString();
    }

    txt.readFields(in);
    st1 = txt.toString();

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.readFields(in);
        dubs[i] = dw.get();
    }

    dw.readFields(in);
    db = dw.get();

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.readFields(in);
        fts[i] = fw.get();
    }

    fw.readFields(in);
    ft = fw.get();

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.readFields(in);
        lngs[i] = lw.get();
    }

    lw.readFields(in);
    lng = lw.get();

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.readFields(in);
        bools[i] = bw.get();
    }

    bw.readFields(in);
    bool = bw.get();

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.readFields(in);
        shrts[i] = sw.get();
    }
    sw.readFields(in);
    shrt = sw.get();

    // 8. bytes
    BytesWritable btsw = new BytesWritable();
    btsw.readFields(in);
    byte[] buffer = btsw.getBytes();
    bts = new byte[btsw.getLength()];
    for (int i = 0; i < btsw.getLength(); i++) {
        bts[i] = buffer[i];
    }
}