cascading.tuple.Tuple Java Examples

private TupleBuilder makeJoinedBuilder( final Fields[] joinFields )
{
	final Fields[] fields = isSelfJoin() ? new Fields[ size() ] : joinFields;

	if( isSelfJoin() ) {
		Arrays.fill(fields, 0, fields.length, joinFields[0]);
	}

	return new TupleBuilder()
	{
		Tuple result = TupleViews.createComposite(fields);

		@Override
		public Tuple makeResult( Tuple[] tuples )
		{
			return TupleViews.reset( result, tuples );
		}
	};
}
 

public static void writeNullMask(
		Tuple t, DataOutputView target) throws IOException {

	final int length = t.size();
	int b;
	int bytePos;

	for(int fieldPos = 0; fieldPos < length; ) {
		b = 0x00;
		// set bits in byte
		for(bytePos = 0; bytePos < 8 && fieldPos < length; bytePos++, fieldPos++) {
			b = b << 1;
			// set bit if field is null
			if(t.getObject(fieldPos) == null) {
				b |= 0x01;
			}
		}
		// shift bits if last byte is not completely filled
		for(; bytePos < 8; bytePos++) {
			b = b << 1;
		}
		// write byte
		target.writeByte(b);
	}
}
 

@Override
public Tuple[] deserialize(Tuple[] reuse, DataInputView source) throws IOException {

	// read null mask
	NullMaskSerDeUtils.readNullMask(this.nullFields, this.fillLength, source);

	// read non-null fields
	for (int i = 0; i < this.fillLength; i++) {

		if(!this.nullFields[i]) {
			reuse[i] = tupleSerializers[i].deserialize(source);
		}
		else {
			reuse[i] = null;
		}
	}
	return reuse;
}
 

@SuppressWarnings("unchecked")
@Override
public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall)
		throws IOException {
	Tuple result = new Tuple();

	Object key = sourceCall.getContext()[0];
	Object value = sourceCall.getContext()[1];
	boolean hasNext = sourceCall.getInput().next(key, value);
	if (!hasNext) {
		return false;
	}

	// Skip nulls
	if (key == null || value == null) {
		return true;
	}

	ImmutableBytesWritable keyWritable = (ImmutableBytesWritable) key;
	Result row = (Result) value;
	result.add(keyWritable);
	result.add(row);
	sourceCall.getIncomingEntry().setTuple(result);
	return true;
}
 

@Test
public void addMultipleTupleEntriesIterable() {
  Fields fields = new Fields("A", "B");
  DataBuilder builder = new DataBuilder(fields);

  List<TupleEntry> tupleEntries = Arrays.asList(new TupleEntry(fields, new Tuple(1, 2)), new TupleEntry(fields,
      new Tuple(3, 4)));

  builder.addTupleEntries(tupleEntries);
  Data source = builder.build();

  List<Tuple> tuples = source.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(1, 2)));
  assertThat(tuples.get(1), is(new Tuple(3, 4)));
}
 

private DataSet<Tuple> translateMerge(List<DataSet<Tuple>> inputs, FlowNode node) {

		DataSet<Tuple> unioned = null;
		TypeInformation<Tuple> type = null;

		int maxDop = -1;

		for(DataSet<Tuple> input : inputs) {
			maxDop = Math.max(maxDop, ((Operator)input).getParallelism());
			if(unioned == null) {
				unioned = input;
				type = input.getType();
			}
			else {
				unioned = unioned.union(input);
			}
		}
		return unioned.map(new IdMapper())
				.returns(type)
				.setParallelism(maxDop);

	}
 

@Override
public void run(Object input) {

	Iterator<Tuple2<Tuple, Tuple[]>> iterator;
	try {
		iterator = (Iterator<Tuple2<Tuple, Tuple[]>>) input;
	}
	catch(ClassCastException cce) {
		throw new RuntimeException("CoGroupInGate requires Iterator<Tuple2<Tuple, Tuple[]>", cce);
	}

	resultIterator.reset(iterator);
	resultIterator.hasNext(); // load first element into closure

	tupleEntryIterator.reset(resultIterator);
	keyEntry.setTuple( this.closure.getGroupTuple(null) );

	next.receive( this, grouping );
}
 

@Override
public Tuple deserialize(Tuple reuse, DataInputView source) throws IOException {

	Tuple tuple = getReuseOrNew(reuse);

	// read null mask
	NullMaskSerDeUtils.readNullMask(nullFields, this.length, source);

	for (int i = 0; i < this.length; i++) {
		Object field;
		if(!this.nullFields[i]) {
			field = fieldSers[i].deserialize(source);
		}
		else {
			field = null;
		}
		tuple.set(i, field);
	}

	return tuple;
}
 

@Test
public void asTupleEntryListWithFields() throws Exception {
  Fields fields = new Fields("A", "B");
  List<Tuple> tuples = new ArrayList<Tuple>();
  tuples.add(new Tuple(1, 100));
  tuples.add(new Tuple(2, 200));

  List<TupleEntry> entryList = new Data(fields, tuples).withFields(new Fields("B")).asTupleEntryList();
  assertThat(entryList.size(), is(2));
  assertThat(entryList.get(0).size(), is(1));
  assertThat(entryList.get(0).getInteger("B"), is(100));
  assertThat(entryList.get(1).size(), is(1));
  assertThat(entryList.get(1).getInteger("B"), is(200));
}
 

@Override
public void reduce(Iterable<Tuple2<Tuple, Tuple[]>> input, Collector<Tuple> output) throws Exception {

	this.streamGraph.setTupleCollector(output);

	if(! this.calledPrepare) {
		this.streamGraph.prepare();
		this.calledPrepare = true;

		this.groupSource.start(this.groupSource);

		this.processBeginTime = System.currentTimeMillis();
		currentProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );
	}

	try {
		this.groupSource.run(input.iterator());
	}
	catch( OutOfMemoryError error ) {
		throw error;
	}
	catch( Throwable throwable ) {
		if( throwable instanceof CascadingException ) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during CoGroupReducer execution", throwable );
	}

}
 

@SuppressWarnings("unchecked")
@Override
public boolean source(FlowProcess<? extends JobConf> fp, SourceCall<Object[], RecordReader> sc)
    throws IOException {
  Container<T> value = (Container<T>) sc.getInput().createValue();
  boolean hasNext = sc.getInput().next(null, value);
  if (!hasNext) { return false; }

  // Skip nulls
  if (value == null) { return true; }

  sc.getIncomingEntry().setTuple(new Tuple(value.get()));
  return true;
}
 

public Tuple peekNextKey() {
	if(peekedValue == null && values.hasNext()) {
		peekedValue = values.next();
		peekedKey = peekedValue.f0;
	}
	if(peekedKey != null) {
		return peekedKey;
	}
	else {
		throw new NoSuchElementException();
	}
}
 

@Override
public Tuple createInstance() {
	try {
		return Tuple.size(0);
	} catch (Exception e) {
		throw new RuntimeException("Cannot instantiate tuple.", e);
	}
}
 

@SuppressWarnings("unchecked")
@Override
public boolean source(FlowProcess<? extends JobConf> fp, SourceCall<Object[], RecordReader> sc)
    throws IOException {
  Container<Tuple> value = (Container<Tuple>) sc.getInput().createValue();
  boolean hasNext = sc.getInput().next(null, value);
  if (!hasNext) { return false; }

  // Skip nulls
  if (value == null) { return true; }

  sc.getIncomingEntry().setTuple(value.get());
  return true;
}
 

@Test(expected = IllegalArgumentException.class)
public void collectIrregularTuple() {
  BufferCallStub<String> stub = new BufferCallStub.Builder<String>(GROUP_FIELDS, NON_GROUP_FIELDS).build();
  assertThat(stub.result().asTupleEntryList().isEmpty(), is(true));

  stub.getOutputCollector().add(new Tuple(1, 2));
}
 

@Test
public void asTupleEntryListWithFieldsNone() throws Exception {
  Fields fields = new Fields("A", "B");
  List<Tuple> tuples = new ArrayList<Tuple>();
  tuples.add(new Tuple(1, 100));
  tuples.add(new Tuple(2, 200));

  List<TupleEntry> entryList = new Data(fields, tuples).withFields(Fields.NONE).asTupleEntryList();
  assertThat(entryList.size(), is(2));
  assertThat(entryList.get(0).size(), is(0));
  assertThat(entryList.get(1).size(), is(0));
}
 

@Override
public void reduce(Iterable<Tuple> input, Collector<Tuple> output) throws Exception {

	this.streamGraph.setTupleCollector(output);

	if(! this.calledPrepare) {
		this.streamGraph.prepare();
		this.calledPrepare = true;

		this.groupSource.start(this.groupSource);

		processBeginTime = System.currentTimeMillis();
		currentProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );
	}


	try {
		this.groupSource.run(input.iterator());
	}
	catch( OutOfMemoryError error ) {
		throw error;
	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException ) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during GroupByReducer execution", throwable );
	}
}
 

@Test
public void addTupleVarargsWithFields() {
  DataBuilder builder = new DataBuilder(new Fields("A", "B"));
  builder.withFields(new Fields("B")).addTuple(1).addTuple(3);
  Data source = builder.build();

  List<Tuple> tuples = source.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(null, 1)));
  assertThat(tuples.get(1), is(new Tuple(null, 3)));
}
 

/** Creates a new tuple for the current group in the stub record sequence. */
public Builder<C> addTuple(Object... values) {
  if (currentGroup == null) {
    throw new IllegalStateException("Must set group before adding tuples.");
  }
  values = FieldTypeValidator.validateValues(nonGroupFields, values);

  currentValues.add(new TupleEntry(nonGroupFields, new Tuple(values)));
  return this;
}
 

@Override
public int extractKeys(Object record, Object[] target, int index) {
	int localIndex = index;
	for(int i = 0; i < comparators.length; i++) {
		localIndex += comparators[i].extractKeys(((Tuple) record).getObject(keyPositions[i]), target, localIndex);
	}
	return localIndex - index;
}
 

@SuppressWarnings("unchecked")
@Override
public void setReference(Tuple toCompare) {
	for (int i = 0; i < this.keyPositions.length; i++) {
		this.comparators[i].setReference(toCompare.getObject(this.keyPositions[i]));
	}
}
 

@Test
public void copyFromTupleSource() {
  Data toCopySource = new DataBuilder(new Fields("A", "B")).addTuple(4, 2).copyTuple().set("A", 1).build();

  Data copied = new DataBuilder(new Fields("A", "B")).copyTuplesFrom(toCopySource).build();

  List<Tuple> tuples = copied.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(4, 2)));
  assertThat(tuples.get(1), is(new Tuple(1, 2)));
}
 

@Test(expected = IllegalArgumentException.class)
public void collectIrregularTupleEntry() {
  stub = new AggregatorCallStub.Builder<String>(GROUP_FIELDS, NON_GROUP_FIELDS).build();
  assertThat(stub.result().asTupleEntryList().isEmpty(), is(true));

  stub.getOutputCollector().add(new TupleEntry(new Fields("X", String.class), new Tuple(1)));
}
 

@Override
public void operate(FlowProcess flowProcess, FunctionCall functionCall) {
  TupleEntry arguments = functionCall.getArguments();
  Tuple result = new Tuple();

  Name name = (Name) arguments.getObject(0);
  result.add(name.getFirst_name());
  result.add(name.getLast_name());
  functionCall.getOutputCollector().add(result);
}
 

@Override
public RecordMaterializer<Tuple> prepareForRead(
    Configuration configuration,
    Map<String, String> keyValueMetaData,
    MessageType fileSchema,
    ReadContext readContext) {
  MessageType requestedSchema = readContext.getRequestedSchema();
  return new TupleRecordMaterializer(requestedSchema);
}
 

/** {@inheritDoc} */
@Override
public boolean source(FlowProcess<? extends Properties> flowProcess, SourceCall<Void, Iterator<Tuple>> sourceCall)
  throws IOException {
  if (sourceCall.getInput().hasNext()) {
    sourceCall.getIncomingEntry().setTuple(sourceCall.getInput().next());
    return true;
  }
  return false;
}
 

@Test(expected = IllegalArgumentException.class)
public void addMultipleTupleEntriesVarArgsInvalidLength() {
  Fields fields = new Fields("A", "B");
  DataBuilder builder = new DataBuilder(fields);
  builder.addTupleEntries(new TupleEntry(fields, new Tuple(1, 2)), new TupleEntry(new Fields("A", "B", "C"),
      new Tuple(1, 2, 3)));
}
 

@SuppressWarnings("unchecked")
@Override
public void setReference(Tuple toCompare) {

	if(!areKeysAbs) {
		makeKeysAbs(keyPositions, toCompare.size());
		areKeysAbs = true;
	}

	for (int i = 0; i < this.keyPositions.length; i++) {
		this.comparators[i].setReference(toCompare.getObject(keyPositions[i]));
	}
}
 

@SuppressWarnings("unchecked")
@Override
public void run(Object input) throws Throwable {

	Iterator<Tuple2<Tuple, Tuple[]>> joinInputIterator;
	try {
		joinInputIterator = (Iterator<Tuple2<Tuple, Tuple[]>>)input;
	}
	catch(ClassCastException cce) {
		throw new RuntimeException("JoinBoundaryInStage requires Iterator<Tuple2<Tuple, Tuple[]>", cce);
	}

	next.start(this);

	while (joinInputIterator.hasNext()) {

		Tuple2<Tuple, Tuple[]> joinListTuple;

		try {
			joinListTuple = joinInputIterator.next();
			flowProcess.increment( StepCounters.Tuples_Read, 1 );
			flowProcess.increment( SliceCounters.Tuples_Read, 1 );
		}
		catch( CascadingException exception ) {
			handleException( exception, null );
			continue;
		}
		catch( Throwable throwable ) {
			handleException( new DuctException( "internal error", throwable ), null );
			continue;
		}

		next.receive( this, joinListTuple );

	}

	next.complete(this);
}
 

@Test
public void asTupleListWithFieldsNone() throws Exception {
  Fields fields = new Fields("A", "B");
  List<Tuple> tuples = new ArrayList<Tuple>();
  tuples.add(new Tuple(1, 100));
  tuples.add(new Tuple(2, 200));

  List<Tuple> entryList = new Data(fields, tuples).withFields(Fields.NONE).asTupleList();
  assertThat(entryList.size(), is(2));
  assertThat(entryList.get(0).size(), is(0));
  assertThat(entryList.get(1).size(), is(0));
}