Java Code Examples for org.apache.hadoop.io.ArrayWritable#set()

@Test
public void testMapWithArrayReadWrite() throws Exception {
    LinkedMapWritable written = new LinkedMapWritable();
    ArrayWritable array = new WritableArrayWritable(Text.class);
    array.set(new Writable[] { new Text("one") , new Text("two"), new Text("three")} );
    written.put(new Text("foo"), array);
    FastByteArrayOutputStream out = new FastByteArrayOutputStream();
    DataOutputStream da = new DataOutputStream(out);
    written.write(da);
    da.close();

    LinkedMapWritable read = new LinkedMapWritable();
    read.readFields(new DataInputStream(new FastByteArrayInputStream(out.bytes())));
    assertThat(read.size(), is(written.size()));
    assertThat(read.toString(), is(written.toString()));
}
 

public static ArrayWritable makeWritable(Collection<?> writables, Class<? extends Writable> impl) {
    Writable[] array = new Writable[writables.size()];
    Iterator<?> writable = writables.iterator();
    for (int i = 0; i < array.length; ++i)
        array[i] = (Writable) writable.next();
    ArrayWritable arrayWritable = new ArrayWritable(impl);
    arrayWritable.set(array);
    return arrayWritable;
}
 

/**
 * Converts a collection of strings into an ArrayWritable and writes it to the given output.
 *
 * @param coll
 * @param output
 * @throws IOException
 */
public static void writeCollection(Collection<String> coll, DataOutput output) throws IOException {
    ArrayWritable aw = new ArrayWritable(Text.class);
    Writable[] writables = new Writable[coll.size()];
    
    Iterator<String> iter = coll.iterator();
    for (int i = 0; i < writables.length; ++i) {
        writables[i] = new Text(iter.next());
    }
    
    aw.set(writables);
    aw.write(output);
}
 

private Pair<Key,Value> makeTop(List<DiscoveredThing> things) {
    Writable[] returnedThings = new Writable[things.size()];
    for (int i = 0; i < returnedThings.length; ++i)
        returnedThings[i] = things.get(i);
    ArrayWritable aw = new ArrayWritable(DiscoveredThing.class);
    aw.set(returnedThings);
    
    DiscoveredThing thing = things.get(0);
    // we want the key to be the last possible key for this date. Return the key as it is in the index (reversed if necessary) to
    // ensure the keys are consistent with the initial seek range.
    String row = (reverseIndex ? new StringBuilder().append(thing.getTerm()).reverse().toString() : thing.getTerm());
    return new Pair<>(new Key(row, thing.getField(), thing.getDate() + '\uffff'), new Value(WritableUtils.toByteArray(aw)));
}
 

/**
*
* Create an empty value
*
* @return value
*/
@Override
public ArrayWritable createValue() {
	ArrayWritable newArrayWritable = new ArrayWritable(SpreadSheetCellDAO.class);
	newArrayWritable.set(new SpreadSheetCellDAO[0]);
	return newArrayWritable;
}
 

/**
*
* Read row from Office document. If document does not match a defined metadata filter then it returns no rows. If no metadata filter is defined or document matches metadata filter then it returns rows, if available in the document/selected sheet
*
* @return true if next more rows are available, false if not
*/
@Override
public boolean next(Text key, ArrayWritable value) throws IOException {
	if (!(this.getOfficeReader().getFiltered())) {
		return false;
	}
	Object[] objectArray = this.getOfficeReader().getNext();
	if (objectArray==null) {
		return false; // no more to read
	}
	SpreadSheetCellDAO[] cellRows = (SpreadSheetCellDAO[])objectArray;
	key.set("["+this.split.getPath().getName()+"]"+this.getOfficeReader().getCurrentSheetName()+"!A"+this.getOfficeReader().getCurrentRow());
	value.set(cellRows);
	return true;	
}
 

@Override
public boolean next(NullWritable key, ArrayWritable value) {
  if (!iterator.hasNext()) {
    return false;
  }
  // Copy from buffer iterator and set to passed writable
  value.set(iterator.next().get());
  return true;
}
 

@Override
public boolean next(NullWritable aVoid, ArrayWritable arrayWritable) throws IOException {
  // Call the underlying parquetReader.next - which may replace the passed in ArrayWritable
  // with a new block of values
  boolean result = this.parquetReader.next(aVoid, arrayWritable);
  if (!result) {
    // if the result is false, then there are no more records
    return false;
  } else {
    // TODO(VC): Right now, we assume all records in log, have a matching base record. (which
    // would be true until we have a way to index logs too)
    // return from delta records map if we have some match.
    String key = arrayWritable.get()[HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS].toString();
    if (deltaRecordMap.containsKey(key)) {
      // TODO(NA): Invoke preCombine here by converting arrayWritable to Avro. This is required since the
      // deltaRecord may not be a full record and needs values of columns from the parquet
      Option<GenericRecord> rec;
      if (usesCustomPayload) {
        rec = deltaRecordMap.get(key).getData().getInsertValue(getWriterSchema());
      } else {
        rec = deltaRecordMap.get(key).getData().getInsertValue(getReaderSchema());
      }
      if (!rec.isPresent()) {
        // If the record is not present, this is a delete record using an empty payload so skip this base record
        // and move to the next record
        return next(aVoid, arrayWritable);
      }
      GenericRecord recordToReturn = rec.get();
      if (usesCustomPayload) {
        // If using a custom payload, return only the projection fields. The readerSchema is a schema derived from
        // the writerSchema with only the projection fields
        recordToReturn = HoodieAvroUtils.rewriteRecordWithOnlyNewSchemaFields(rec.get(), getReaderSchema());
      }
      // we assume, a later safe record in the log, is newer than what we have in the map &
      // replace it. Since we want to return an arrayWritable which is the same length as the elements in the latest
      // schema, we use writerSchema to create the arrayWritable from the latest generic record
      ArrayWritable aWritable = (ArrayWritable) HoodieRealtimeRecordReaderUtils.avroToArrayWritable(recordToReturn, getHiveSchema());
      Writable[] replaceValue = aWritable.get();
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("key %s, base values: %s, log values: %s", key, HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable),
            HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable)));
      }
      Writable[] originalValue = arrayWritable.get();
      try {
        System.arraycopy(replaceValue, 0, originalValue, 0, originalValue.length);
        arrayWritable.set(originalValue);
      } catch (RuntimeException re) {
        LOG.error("Got exception when doing array copy", re);
        LOG.error("Base record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable));
        LOG.error("Log record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable));
        String errMsg = "Base-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable)
            + " ,Log-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable) + " ,Error :" + re.getMessage();
        throw new RuntimeException(errMsg, re);
      }
    }
    return true;
  }
}