org.apache.parquet.io.api.RecordConsumer Java Examples

The following examples show how to use org.apache.parquet.io.api.RecordConsumer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testRepeatedIntMessageEmptySpecsCompliant() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.RepeatedIntMessage.class, readConsumerMock, conf);

  TestProtobuf.RepeatedIntMessage.Builder msg = TestProtobuf.RepeatedIntMessage.newBuilder();

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #2

Source File: FixedLenDecimalParquetValueWriter.java From Bats with Apache License 2.0

6 votes

@Override
public void writeValue(RecordConsumer consumer, DrillBuf buffer, int start, int end, int precision) {
  int typeLength = DecimalUtility.getMaxBytesSizeForPrecision(precision);
  int length = end - start;
  int startPos = typeLength - length;
  byte[] output = new byte[typeLength];
  if (startPos >= 0) {
    buffer.getBytes(start, output, startPos, length);
    if (output[startPos] < 0) {
      Arrays.fill(output, 0, output.length - length, (byte) -1);
    }
  } else {
    // in this case value from FIXED_LEN_BYTE_ARRAY or BINARY field with greater length was taken, ignore leading bytes
    buffer.getBytes(start - startPos, output, 0, length + startPos);
  }
  consumer.addBinary(Binary.fromReusedByteArray(output));
}

Example #3

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testRepeatedIntMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.RepeatedIntMessage.class, readConsumerMock);

  TestProtobuf.RepeatedIntMessage.Builder msg = TestProtobuf.RepeatedIntMessage.newBuilder();
  msg.addRepeatedInt(1323);
  msg.addRepeatedInt(54469);

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).addInteger(1323);
  inOrder.verify(readConsumerMock).addInteger(54469);
  inOrder.verify(readConsumerMock).endField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #4

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testOptionalInnerMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.MessageA.class, readConsumerMock);

  TestProtobuf.MessageA.Builder msg = TestProtobuf.MessageA.newBuilder();
  msg.getInnerBuilder().setOne("one");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("inner", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("one".getBytes()));
  inOrder.verify(readConsumerMock).endField("one", 0);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("inner", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #5

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testSimplestMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.InnerMessage.class, readConsumerMock);

  TestProtobuf.InnerMessage.Builder msg = TestProtobuf.InnerMessage.newBuilder();
  msg.setOne("oneValue");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromString("oneValue"));
  inOrder.verify(readConsumerMock).endField("one", 0);

  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #6

Source File: PentahoParquetWriteSupport.java From pentaho-hadoop-shims with Apache License 2.0

6 votes

public void writeRow( RowMetaAndData row, RecordConsumer consumer ) {
  consumer.startMessage();
  int index = 0;
  for ( IParquetOutputField f : outputFields ) {
    if ( f.getFormatFieldName() == null ) {
      continue;
    }
    try {
      writeField( f, index, row, consumer );
      index++;
    } catch ( KettleValueException ex ) {
      throw new RuntimeException( ex );
    }
  }
  consumer.endMessage();
}

Example #7

Source File: TestColumnIO.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testEmptyField() {
  MemPageStore memPageStore = new MemPageStore(1);
  ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
  MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
  final RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
  recordWriter.startMessage();
  recordWriter.startField("DocId", 0);
  recordWriter.addLong(0);
  recordWriter.endField("DocId", 0);
  recordWriter.startField("Links", 1);
  try {
    recordWriter.endField("Links", 1);
    Assert.fail("expected exception because of empty field");
  } catch (ParquetEncodingException e) {
    Assert.assertEquals("empty fields are illegal, the field should be ommited completely instead", e.getMessage());
  }
}

Example #8

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testProto3RepeatedIntMessageEmptySpecsCompliant() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.RepeatedIntMessage.class, readConsumerMock, conf);

  TestProto3.RepeatedIntMessage.Builder msg = TestProto3.RepeatedIntMessage.newBuilder();

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #9

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testProto3RepeatedIntMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.RepeatedIntMessage.class, readConsumerMock);

  TestProto3.RepeatedIntMessage.Builder msg = TestProto3.RepeatedIntMessage.newBuilder();
  msg.addRepeatedInt(1323);
  msg.addRepeatedInt(54469);

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).addInteger(1323);
  inOrder.verify(readConsumerMock).addInteger(54469);
  inOrder.verify(readConsumerMock).endField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #10

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testProto3MapIntMessageEmptySpecsCompliant() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.MapIntMessage.class, readConsumerMock, conf);

  TestProto3.MapIntMessage.Builder msg = TestProto3.MapIntMessage.newBuilder();
  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #11

Source File: ParquetRowDataWriter.java From flink with Apache License 2.0

5 votes

public ParquetRowDataWriter(
		RecordConsumer recordConsumer,
		RowType rowType,
		GroupType schema,
		boolean utcTimestamp) {
	this.recordConsumer = recordConsumer;
	this.utcTimestamp = utcTimestamp;

	this.filedWriters = new FieldWriter[rowType.getFieldCount()];
	this.fieldNames = rowType.getFieldNames().toArray(new String[0]);
	for (int i = 0; i < rowType.getFieldCount(); i++) {
		this.filedWriters[i] = createWriter(rowType.getTypeAt(i), schema.getType(i));
	}
}

Example #12

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testRepeatedInnerMessageMessage_message() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.TopMessage.class, readConsumerMock);

  TestProtobuf.TopMessage.Builder msg = TestProtobuf.TopMessage.newBuilder();
  msg.addInnerBuilder().setOne("one").setTwo("two");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("inner", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("one".getBytes()));
  inOrder.verify(readConsumerMock).endField("one", 0);
  inOrder.verify(readConsumerMock).startField("two", 1);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("two".getBytes()));
  inOrder.verify(readConsumerMock).endField("two", 1);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("inner", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #13

Source File: TestColumnIO.java From parquet-mr with Apache License 2.0

5 votes

private void writeGroups(MessageType writtenSchema, MemPageStore memPageStore, Group... groups) {
  ColumnIOFactory columnIOFactory = new ColumnIOFactory(true);
  ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
  MessageColumnIO columnIO = columnIOFactory.getColumnIO(writtenSchema);
  RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
  GroupWriter groupWriter = new GroupWriter(recordWriter, writtenSchema);
  for (Group group : groups) {
    groupWriter.write(group);
  }
  recordWriter.flush();
  columns.flush();
}

Example #14

Source File: TestColumnIO.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testPushParser() {
  MemPageStore memPageStore = new MemPageStore(1);
  ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
  MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
  RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
  new GroupWriter(recordWriter, schema).write(r1);
  recordWriter.flush();
  columns.flush();

  RecordReader<Void> recordReader = columnIO.getRecordReader(memPageStore, new ExpectationValidatingConverter(expectedEventsForR1, schema));
  recordReader.read();

}

Example #15

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testRepeatedInnerMessageSpecsCompliantMessage_message() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.TopMessage.class, readConsumerMock, conf);

  TestProtobuf.TopMessage.Builder msg = TestProtobuf.TopMessage.newBuilder();
  msg.addInnerBuilder().setOne("one").setTwo("two");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("inner", 0);
  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("list", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("element", 0);
  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("one".getBytes()));
  inOrder.verify(readConsumerMock).endField("one", 0);
  inOrder.verify(readConsumerMock).startField("two", 1);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("two".getBytes()));
  inOrder.verify(readConsumerMock).endField("two", 1);
  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("element", 0);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("list", 0);
  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("inner", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #16

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testProto3RepeatedIntMessageEmpty() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.RepeatedIntMessage.class, readConsumerMock);

  TestProto3.RepeatedIntMessage.Builder msg = TestProto3.RepeatedIntMessage.newBuilder();

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #17

Source File: TestArrayCompatibility.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testRepeatedPrimitiveInList() throws Exception {
  Path test = writeDirect(
      "message RepeatedPrimitiveInList {" +
          "  required group list_of_ints (LIST) {" +
          "    repeated int32 array;" +
          "  }" +
          "}",
      new DirectWriter() {
        @Override
        public void write(RecordConsumer rc) {
          rc.startMessage();
          rc.startField("list_of_ints", 0);

          rc.startGroup();
          rc.startField("array", 0);

          rc.addInteger(34);
          rc.addInteger(35);
          rc.addInteger(36);

          rc.endField("array", 0);
          rc.endGroup();

          rc.endField("list_of_ints", 0);
          rc.endMessage();
        }
      });

  ListOfInts expected = new ListOfInts(Lists.newArrayList(34, 35,36));
  ListOfInts actual = reader(test, ListOfInts.class).read();
  Assert.assertEquals("Should read record correctly", expected, actual);
}

Example #18

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test(expected = UnsupportedOperationException.class)
public void testMessageWithExtensions() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.Vehicle.class, readConsumerMock);

  TestProtobuf.Vehicle.Builder msg = TestProtobuf.Vehicle.newBuilder();
  msg.setHorsePower(300);
  // Currently there's no support for extension fields. This test tests that the extension field
  // will cause an exception.
  msg.setExtension(TestProtobuf.Airplane.wingSpan, 50);

  instance.write(msg.build());
}

Example #19

Source File: TestColumnIO.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testWriteWithGroupWriter() {

  final String[] expected = {
      "[DocId]: 10, r:0, d:0",
      "[Links, Forward]: 20, r:0, d:2",
      "[Links, Forward]: 40, r:1, d:2",
      "[Links, Forward]: 60, r:1, d:2",
      "[Links, Backward]: null, r:0, d:1",
      "[Name, Language, Code]: en-us, r:0, d:2",
      "[Name, Language, Country]: us, r:0, d:3",
      "[Name, Language, Code]: en, r:2, d:2",
      "[Name, Language, Country]: null, r:2, d:2",
      "[Name, Url]: http://A, r:0, d:2",
      "[Name, Url]: http://B, r:1, d:2",
      "[Name, Language, Code]: null, r:1, d:1",
      "[Name, Language, Country]: null, r:1, d:1",
      "[Name, Language, Code]: en-gb, r:1, d:2",
      "[Name, Language, Country]: gb, r:1, d:3",
      "[Name, Url]: null, r:1, d:1",
      "[DocId]: 20, r:0, d:0",
      "[Links, Backward]: 10, r:0, d:2",
      "[Links, Backward]: 30, r:1, d:2",
      "[Links, Forward]: 80, r:0, d:2",
      "[Name, Url]: http://C, r:0, d:2",
      "[Name, Language, Code]: null, r:0, d:1",
      "[Name, Language, Country]: null, r:0, d:1"
  };

  ValidatingColumnWriteStore columns = new ValidatingColumnWriteStore(expected);
  MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
  RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
  GroupWriter groupWriter = new GroupWriter(recordWriter, schema);
  groupWriter.write(r1);
  groupWriter.write(r2);
  recordWriter.flush();
  columns.validate();
  columns.flush();
  columns.close();
}

Example #20

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testRepeatedIntMessageEmpty() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.RepeatedIntMessage.class, readConsumerMock);

  TestProtobuf.RepeatedIntMessage.Builder msg = TestProtobuf.RepeatedIntMessage.newBuilder();

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #21

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testProto3OptionalInnerMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.MessageA.class, readConsumerMock);

  TestProto3.MessageA.Builder msg = TestProto3.MessageA.newBuilder();
  msg.getInnerBuilder().setOne("one");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("inner", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("one".getBytes()));
  inOrder.verify(readConsumerMock).endField("one", 0);
  inOrder.verify(readConsumerMock).startField("two", 1);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("".getBytes()));
  inOrder.verify(readConsumerMock).endField("two", 1);
  inOrder.verify(readConsumerMock).startField("three", 2);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("".getBytes()));
  inOrder.verify(readConsumerMock).endField("three", 2);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("inner", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #22

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testProto3MapIntMessage() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.MapIntMessage.class, readConsumerMock);

  TestProto3.MapIntMessage.Builder msg = TestProto3.MapIntMessage.newBuilder();
  msg.putMapInt(123, 1);
  msg.putMapInt(234, 2);
  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("mapInt", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("key", 0);
  inOrder.verify(readConsumerMock).addInteger(123);
  inOrder.verify(readConsumerMock).endField("key", 0);
  inOrder.verify(readConsumerMock).startField("value", 1);
  inOrder.verify(readConsumerMock).addInteger(1);
  inOrder.verify(readConsumerMock).endField("value", 1);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("key", 0);
  inOrder.verify(readConsumerMock).addInteger(234);
  inOrder.verify(readConsumerMock).endField("key", 0);
  inOrder.verify(readConsumerMock).startField("value", 1);
  inOrder.verify(readConsumerMock).addInteger(2);
  inOrder.verify(readConsumerMock).endField("value", 1);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("mapInt", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #23

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testMapIntMessageEmpty() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProtobuf.MapIntMessage.class, readConsumerMock);

  TestProtobuf.MapIntMessage.Builder msg = TestProtobuf.MapIntMessage.newBuilder();
  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #24

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

5 votes

@Test
public void testProto3RepeatedIntMessageSpecsCompliant() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.RepeatedIntMessage.class, readConsumerMock, conf);

  TestProto3.RepeatedIntMessage.Builder msg = TestProto3.RepeatedIntMessage.newBuilder();
  msg.addRepeatedInt(1323);
  msg.addRepeatedInt(54469);

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("list", 0);

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("element", 0);
  inOrder.verify(readConsumerMock).addInteger(1323);
  inOrder.verify(readConsumerMock).endField("element", 0);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("element", 0);
  inOrder.verify(readConsumerMock).addInteger(54469);
  inOrder.verify(readConsumerMock).endField("element", 0);
  inOrder.verify(readConsumerMock).endGroup();

  inOrder.verify(readConsumerMock).endField("list", 0);
  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("repeatedInt", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}

Example #25

Source File: ParquetRowDataBuilder.java From flink with Apache License 2.0

5 votes

@Override
public void prepareForWrite(RecordConsumer recordConsumer) {
	this.writer = new ParquetRowDataWriter(
			recordConsumer,
			rowType,
			schema,
			utcTimestamp);
}

Example #26

Source File: TupleWriteSupport.java From parquet-mr with Apache License 2.0

4 votes

@Override
public void prepareForWrite(RecordConsumer recordConsumer) {
  this.recordConsumer = recordConsumer;
}

Example #27

Source File: RecordReaderImplementation.java From parquet-mr with Apache License 2.0

4 votes

private RecordConsumer wrap(RecordConsumer recordConsumer) {
  if (LOG.isDebugEnabled()) {
    return new RecordConsumerLoggingWrapper(recordConsumer);
  }
  return recordConsumer;
}

Example #28

Source File: TupleWriteSupport.java From parquet-mr with Apache License 2.0

4 votes

@Override
public void prepareForWrite(RecordConsumer recordConsumer) {
  this.recordConsumer = recordConsumer;
}

Example #29

Source File: ThriftWriteSupport.java From parquet-mr with Apache License 2.0

4 votes

@Override
public void prepareForWrite(RecordConsumer recordConsumer) {
  this.writeSupport.prepareForWrite(recordConsumer);
}

Example #30

Source File: ProtoWriteSupportTest.java From parquet-mr with Apache License 2.0

4 votes

@Test
public void testProto3RepeatedInnerMessageSpecsCompliantMessage_message() throws Exception {
  RecordConsumer readConsumerMock =  Mockito.mock(RecordConsumer.class);
  Configuration conf = new Configuration();
  ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
  ProtoWriteSupport instance = createReadConsumerInstance(TestProto3.TopMessage.class, readConsumerMock, conf);

  TestProto3.TopMessage.Builder msg = TestProto3.TopMessage.newBuilder();
  msg.addInnerBuilder().setOne("one").setTwo("two");

  instance.write(msg.build());

  InOrder inOrder = Mockito.inOrder(readConsumerMock);

  inOrder.verify(readConsumerMock).startMessage();
  inOrder.verify(readConsumerMock).startField("inner", 0);
  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("list", 0);
  inOrder.verify(readConsumerMock).startGroup();

  inOrder.verify(readConsumerMock).startField("element", 0);
  inOrder.verify(readConsumerMock).startGroup();
  inOrder.verify(readConsumerMock).startField("one", 0);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("one".getBytes()));
  inOrder.verify(readConsumerMock).endField("one", 0);
  inOrder.verify(readConsumerMock).startField("two", 1);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("two".getBytes()));
  inOrder.verify(readConsumerMock).endField("two", 1);
  inOrder.verify(readConsumerMock).startField("three", 2);
  inOrder.verify(readConsumerMock).addBinary(Binary.fromConstantByteArray("".getBytes()));
  inOrder.verify(readConsumerMock).endField("three", 2);
  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("element", 0);

  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("list", 0);
  inOrder.verify(readConsumerMock).endGroup();
  inOrder.verify(readConsumerMock).endField("inner", 0);
  inOrder.verify(readConsumerMock).endMessage();
  Mockito.verifyNoMoreInteractions(readConsumerMock);
}