cascading.tuple.TupleEntry Java Examples

@Test
public void completeDifferentOutputFields() {
  List<TupleEntry> actual = new BufferCallStub.Builder<Void>(GROUP_FIELDS, NON_GROUP_FIELDS)
      .outputFields(OUTPUT)
      .newGroup(1)
      .addTuple("a")
      .addTuple("b")
      .newGroup(2)
      .addTuple("c")
      .addTuple("d")
      .build()
      .complete(mock(FlowProcess.class), new CountBuffer())
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(4));
  assertThat(actual.get(0), tupleEntry(OUTPUT, 1));
  assertThat(actual.get(1), tupleEntry(OUTPUT, 2));
  assertThat(actual.get(2), tupleEntry(OUTPUT, 1));
  assertThat(actual.get(3), tupleEntry(OUTPUT, 2));
}
 

public SinkBoundaryInStage(FlowProcess flowProcess, FlowElement flowElement, FlowNode node) {
	super(flowProcess, flowElement);
	this.nextStarted = false;

	Scope inScope = node.getElementGraph().incomingEdgesOf(flowElement).iterator().next();

	Fields inFields;
	if(inScope.isEvery()) {
		inFields = inScope.getOutGroupingFields();
	}
	else {
		inFields = inScope.getOutValuesFields();
	}

	this.tupleEntry = new TupleEntry(inFields);
}
 

@Override
public void write(TupleEntry record) {
  recordConsumer.startMessage();
  final List<Type> fields = rootSchema.getFields();

  for (int i = 0; i < fields.size(); i++) {
    Type field = fields.get(i);

    if (record == null || record.getObject(field.getName()) == null) {
      continue;
    }
    recordConsumer.startField(field.getName(), i);
    if (field.isPrimitive()) {
      writePrimitive(record, field.asPrimitiveType());
    } else {
      throw new UnsupportedOperationException("Complex type not implemented");
    }
    recordConsumer.endField(field.getName(), i);
  }
  recordConsumer.endMessage();
}
 

private void writeToHadoopPartitionTap(Tap<?, ?, ?> tap) throws IOException {
  @SuppressWarnings("unchecked")
  BasePartitionTap<JobConf, ?, ?> hadoopTap = (BasePartitionTap<JobConf, ?, ?>) tap;
  JobConf conf = new JobConf();

  // Avoids deletion of results when using a partition tap (close() will delete the _temporary before the copy has
  // been done if not in a flow)
  HadoopUtil.setIsInflow(conf);

  HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf);
  hadoopTap.sinkConfInit(flowProcess, conf);
  TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess);
  for (TupleEntry tuple : data.asTupleEntryList()) {
    collector.add(tuple);
  }
  collector.close();

  // We need to clean up the '_temporary' folder
  BasePartitionTap<JobConf, ?, ?> partitionTap = hadoopTap;
  @SuppressWarnings("unchecked")
  String basePath = partitionTap.getParent().getFullIdentifier(flowProcess);
  deleteTemporaryPath(new Path(basePath), FileSystem.get(conf));
}
 

private void writeToLocalTap(Tap<?, ?, ?> tap) throws IOException {
  @SuppressWarnings("unchecked")
  Tap<Properties, ?, ?> localTap = (Tap<Properties, ?, ?>) tap;
  Properties conf = new Properties();
  LocalFlowProcess flowProcess = new LocalFlowProcess(conf);

  flowProcess.setStepStats(new LocalStepStats(new NullFlowStep(), NullClientState.INSTANCE));

  localTap.sinkConfInit(flowProcess, conf);
  TupleEntryCollector collector = localTap.openForWrite(flowProcess);
  for (TupleEntry tuple : data.asTupleEntryList()) {
    collector.add(tuple);
  }
  collector.close();
  localTap.commitResource(conf);
}
 

@Test
public void completeDifferentOutputFields() {
  @SuppressWarnings({ "unchecked", "rawtypes" })
  List<TupleEntry> actual = new AggregatorCallStub.Builder(GROUP_FIELDS, NON_GROUP_FIELDS)
      .outputFields(OUTPUT_FIELDS)
      .newGroup(1)
      .addTuple("a")
      .addTuple("b")
      .newGroup(2)
      .addTuple("c")
      .addTuple("d")
      .build()
      .complete(mock(FlowProcess.class), new MaxValue(OUTPUT_FIELDS))
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(2));
  assertThat(actual.get(0), tupleEntry(OUTPUT_FIELDS, "b"));
  assertThat(actual.get(1), tupleEntry(OUTPUT_FIELDS, "d"));
}
 

@Test
public void completeDifferentOutputFields() {
  @SuppressWarnings("unchecked")
  List<TupleEntry> actual = new FunctionCallStub.Builder<Void>(FIELDS)
      .outputFields(OUTPUT)
      .addTuple("a")
      .addTuple("b")
      .build()
      .complete(mock(FlowProcess.class), new Insert(OUTPUT, 1))
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(2));
  assertThat(actual.get(0), tupleEntry(OUTPUT, 1));
  assertThat(actual.get(1), tupleEntry(OUTPUT, 1));
}
 

@Test
public void complete() {
  List<TupleEntry> actual = new AggregatorCallStub.Builder<Tuple[]>(GROUP_FIELDS, NON_GROUP_FIELDS)
      .newGroup(1)
      .addTuple("a")
      .addTuple("b")
      .newGroup(2)
      .addTuple("c")
      .addTuple("d")
      .build()
      .complete(mock(FlowProcess.class), new First(NON_GROUP_FIELDS))
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(2));
  assertThat(actual.get(0), tupleEntry(NON_GROUP_FIELDS, "a"));
  assertThat(actual.get(1), tupleEntry(NON_GROUP_FIELDS, "c"));
}
 

@Test
public void addMultipleTupleEntriesIterable() {
  Fields fields = new Fields("A", "B");
  DataBuilder builder = new DataBuilder(fields);

  List<TupleEntry> tupleEntries = Arrays.asList(new TupleEntry(fields, new Tuple(1, 2)), new TupleEntry(fields,
      new Tuple(3, 4)));

  builder.addTupleEntries(tupleEntries);
  Data source = builder.build();

  List<Tuple> tuples = source.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(1, 2)));
  assertThat(tuples.get(1), is(new Tuple(3, 4)));
}
 

@Test
public void selectFieldsToSetUsingMultipleEntriesIterableInsert() {
  Fields fields = new Fields("A", "B", "C", "D");
  DataBuilder builder = new DataBuilder(fields);
  Fields subFields = new Fields("B", "D");

  List<TupleEntry> tupleEntries = Arrays.asList(new TupleEntry(subFields, new Tuple(1, 2)), new TupleEntry(subFields,
      new Tuple(3, 4)));

  builder.withFields(subFields).addTupleEntries(tupleEntries);
  Data source = builder.build();

  List<Tuple> tuples = source.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(null, 1, null, 2)));
  assertThat(tuples.get(1), is(new Tuple(null, 3, null, 4)));
}
 

@Test
public void complete() {
  @SuppressWarnings("unchecked")
  List<TupleEntry> actual = new BufferCallStub.Builder<Void>(GROUP_FIELDS, NON_GROUP_FIELDS)
      .newGroup(1)
      .addTuple("a")
      .addTuple("b")
      .newGroup(2)
      .addTuple("c")
      .addTuple("d")
      .build()
      .complete(mock(FlowProcess.class), new FirstNBuffer(1))
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(2));
  assertThat(actual.get(0), tupleEntry(NON_GROUP_FIELDS, "a"));
  assertThat(actual.get(1), tupleEntry(NON_GROUP_FIELDS, "c"));
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void complete() {
  List<TupleEntry> actual = new FunctionCallStub.Builder(FIELDS)
      .addTuple("a")
      .addTuple("b")
      .build()
      .complete(mock(FlowProcess.class), new Identity())
      .result()
      .asTupleEntryList();

  assertThat(actual.size(), is(2));
  assertThat(actual.get(0), tupleEntry(FIELDS, "a"));
  assertThat(actual.get(1), tupleEntry(FIELDS, "b"));
}
 

@Test
public void collectTupleEntryCopiesCollectedTupleEntries() throws Exception {
  MockOperation operation = new MockOperation();
  TupleEntry tupleEntry = new TupleEntry(declaredField, new Tuple("value1"));
  operation.getOutputCollector().add(tupleEntry);
  tupleEntry.setString(declaredField, "value2");
  operation.getOutputCollector().add(tupleEntry);
  List<TupleEntry> tupleEntries = operation.result().asTupleEntryList();
  assertThat(tupleEntries.size(), is(2));
  assertThat(tupleEntries.get(0).getString(declaredField), is("value1"));
  assertThat(tupleEntries.get(1).getString(declaredField), is("value2"));
}
 

@Override
public void operate(@SuppressWarnings("rawtypes") FlowProcess flowProcess, BufferCall<Void> bufferCall) {
  Iterator<TupleEntry> iterator = bufferCall.getArgumentsIterator();
  int count = 0;
  while (iterator.hasNext()) {
    iterator.next();
    bufferCall.getOutputCollector().add(new Tuple(++count));
  }
}
 

@Test
public void addMultipleTupleEntriesVarArgs() {
  Fields fields = new Fields("A", "B");
  DataBuilder builder = new DataBuilder(fields);
  builder.addTupleEntries(new TupleEntry(fields, new Tuple(1, 2)), new TupleEntry(fields, new Tuple(3, 4)));
  Data source = builder.build();

  List<Tuple> tuples = source.getTuples();

  assertThat(tuples.size(), is(2));
  assertThat(tuples.get(0), is(new Tuple(1, 2)));
  assertThat(tuples.get(1), is(new Tuple(3, 4)));
}
 

@Test(expected = IllegalArgumentException.class)
public void addMultipleTupleEntriesIterableInvalidLength() {
  DataBuilder builder = new DataBuilder(new Fields("A", "B"));

  List<TupleEntry> tupleEntries = Arrays.asList(new TupleEntry(new Fields("A", "B"), new Tuple(1, 2)),
      new TupleEntry(new Fields("A", "B", "C"), new Tuple(1, 2, 3)));

  builder.addTupleEntries(tupleEntries);
}
 

/** Creates a new tuple for the current group in the stub record sequence. */
public Builder<C> addTuple(Object... values) {
  if (currentGroup == null) {
    throw new IllegalStateException("Must set group before adding tuples.");
  }
  values = FieldTypeValidator.validateValues(nonGroupFields, values);

  currentValues.add(new TupleEntry(nonGroupFields, new Tuple(values)));
  return this;
}
 

/** Creates a new group in the stub record sequence. */
public Builder<C> newGroup(Object... values) {
  values = FieldTypeValidator.validateValues(groupFields, values);
  flush();
  currentGroup = new TupleEntry(groupFields, new Tuple(values));
  currentValues = new ArrayList<TupleEntry>();
  return this;
}
 

/** Advances to the next group */
public AggregatorCallStub<C> nextGroup() {
  Entry<TupleEntry, List<TupleEntry>> next = groupsIterator.next();
  currentGroup = next.getKey();
  valuesIterator = next.getValue().iterator();
  return this;
}
 

@SuppressWarnings("unchecked")
@Override
public void sink(FlowProcess<JobConf> fp, SinkCall<Object[], OutputCollector> sc)
    throws IOException {
  TupleEntry tuple = sc.getOutgoingEntry();

  if (tuple.size() != 1) {
    throw new RuntimeException("ParquetValueScheme expects tuples with an arity of exactly 1, but found " + tuple.getFields());
  }

  T value = (T) tuple.getObject(0);
  OutputCollector output = sc.getOutput();
  output.collect(null, value);
}
 

@Test
public void asTupleEntryList() throws IOException {
  Bucket sink = new Bucket(FIELDS, pipe, flow);
  TupleEntryCollector collector = sink.openForWrite(null, null);
  collector.add(TUPLE_1);
  collector.add(TUPLE_2);
  List<TupleEntry> tupleEntryList = sink.result().asTupleEntryList();
  assertThat(tupleEntryList.size(), is(2));
  assertThat(tupleEntryList.get(0).getFields(), is(FIELDS));
  assertThat(tupleEntryList.get(0).getTuple(), is(TUPLE_1));
  assertThat(tupleEntryList.get(1).getFields(), is(FIELDS));
  assertThat(tupleEntryList.get(1).getTuple(), is(TUPLE_2));
}
 

public Builder<C> addTuple(Object... values) {
  values = FieldTypeValidator.validateValues(fieldMask, values);
  TupleEntry newTuple = new TupleEntry(fields, Tuple.size(fields.size()));
  newTuple.setTuple(fieldMask, new Tuple(values));
  tuples.add(newTuple);
  return this;
}
 

/**
 * Returns the data captured by this stub as a {@link Data} instance which enables further sorting, filtering, and
 * transformation.
 */
public Data result() {
  List<Tuple> output = new ArrayList<Tuple>();
  for (TupleEntry entry : collected) {
    output.add(entry.getTupleCopy());
  }
  return new Data(declaredFields, Collections.unmodifiableList(output));
}
 

@Override
protected void collect(TupleEntry tupleEntry) throws IOException {
  if (!tupleEntry.getFields().equals(getDeclaredFields())) {
    throw new IllegalArgumentException("Collected fields != declared fields: " + tupleEntry.getFields() + ", "
        + getDeclaredFields());
  }
  collected.add(new TupleEntry(tupleEntry));
}
 

@Override
public void add(Tuple tuple) {
  if (tuple.size() != getDeclaredFields().size()) {
    throw new IllegalArgumentException("Tuple size != declared fields size: " + tuple + ", " + getDeclaredFields());
  }
  collected.add(new TupleEntry(getDeclaredFields(), new Tuple(tuple)));
}
 

/** Copies the {@link Tuple Tuples} from the provided {@link Data} into this {@link DataBuilder}. */
public DataBuilder copyTuplesFrom(Data source) {
  for (Tuple tuple : source.getTuples()) {
    flushTupleEntry();
    tupleEntry = new TupleEntry(fields, new Tuple(tuple));
  }
  return this;
}
 

/** Makes a copy of the current {@link Tuple}. */
public DataBuilder copyTuple() {
  TupleEntry copy = new TupleEntry(tupleEntry);
  flushTupleEntry();
  tupleEntry = copy;
  return this;
}
 

/** Creates a new List of {@link TupleEntry} with the specified values. */
public DataBuilder addTupleEntries(TupleEntry... entries) {
  for (TupleEntry entry : entries) {
    addTupleEntry(entry);
  }
  return this;
}
 

/** Creates a new List of {@link TupleEntry} with the specified values. */
public DataBuilder addTupleEntries(Iterable<TupleEntry> entries) {
  for (TupleEntry entry : entries) {
    addTupleEntry(entry);
  }
  return this;
}
 

@Test
public void openForRead() throws IOException {
  TupleEntryIterator iterator = tap.openForRead(null, null);
  assertThat(iterator.hasNext(), is(true));
  assertThat(iterator.next(), is(new TupleEntry(FIELDS, TUPLE_1)));
  assertThat(iterator.hasNext(), is(true));
  assertThat(iterator.next(), is(new TupleEntry(FIELDS, TUPLE_2)));
  assertThat(iterator.hasNext(), is(false));
}