Java Code Examples for org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName

The following examples show how to use org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: parquet-mr   Source File: TestTypeBuilders.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBinaryAnnotationsRejectsNonBinary() {
  OriginalType[] types = new OriginalType[] {
      UTF8, JSON, BSON};
  for (final OriginalType logicalType : types) {
    PrimitiveTypeName[] nonBinary = new PrimitiveTypeName[]{
        BOOLEAN, INT32, INT64, INT96, DOUBLE, FLOAT
    };
    for (final PrimitiveTypeName type : nonBinary) {
      assertThrows("Should reject non-binary type: " + type,
          IllegalStateException.class, (Callable<Type>) () -> Types.required(type).as(logicalType).named("col"));
    }
    assertThrows("Should reject non-binary type: FIXED_LEN_BYTE_ARRAY",
        IllegalStateException.class, (Callable<Type>) () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
            .as(logicalType).named("col"));
  }
}
 
Example 2
@Test
public void testInt32AnnotationsRejectNonInt32() {
  LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
    dateType(), timeType(true, MILLIS), timeType(false, MILLIS),
    intType(8, false), intType(16, false), intType(32, false),
    intType(8, true), intType(16, true), intType(32, true)};
  for (final LogicalTypeAnnotation logicalType : types) {
    PrimitiveTypeName[] nonInt32 = new PrimitiveTypeName[]{
        BOOLEAN, INT64, INT96, DOUBLE, FLOAT, BINARY
    };
    for (final PrimitiveTypeName type : nonInt32) {
      assertThrows("Should reject non-int32 type: " + type,
          IllegalStateException.class, () -> Types.required(type).as(logicalType).named("col"));
    }
    assertThrows("Should reject non-int32 type: FIXED_LEN_BYTE_ARRAY",
        IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
            .as(logicalType).named("col"));
  }
}
 
Example 3
Source Project: dremio-oss   Source File: ColumnReader.java    License: Apache License 2.0 6 votes vote down vote up
protected ColumnReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                       ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
  this.parentReader = parentReader;
  this.columnDescriptor = descriptor;
  this.columnChunkMetaData = columnChunkMetaData;
  this.isFixedLength = fixedLength;
  this.schemaElement = schemaElement;
  this.valueVec =  v;
  this.pageReader = (parentReader.getSingleStream() != null)?
    new DeprecatedSingleStreamPageReader(this, parentReader.getSingleStream(), parentReader.getFsPath(), columnChunkMetaData) :
    new PageReader(this, parentReader.getFileSystem(), parentReader.getFsPath(), columnChunkMetaData);

  if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
    if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
    } else if (columnDescriptor.getType() == PrimitiveTypeName.INT96
      && valueVec instanceof TimeStampMilliVector) {
      // if int 96 column is being read as a Timestamp, this truncates the time format used by Impala
      // dataTypeLengthInBits is only ever used when computing offsets into the destination vector, so it
      // needs to be set to the bit width of the resulting Arrow type, usually this matches the input length
      dataTypeLengthInBits = 64;
    } else {
      dataTypeLengthInBits = DeprecatedParquetVectorizedReader.getTypeLengthInBits(columnDescriptor.getType());
    }
  }
}
 
Example 4
Source Project: parquet-mr   Source File: TestParquetMetadataConverter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIgnoreStatsWithSignedSortOrder() {
  ParquetMetadataConverter converter = new ParquetMetadataConverter();
  BinaryStatistics stats = new BinaryStatistics();
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("z"));
  stats.incrementNumNulls();

  PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY)
      .as(OriginalType.UTF8).named("b");
  Statistics convertedStats = converter.fromParquetStatistics(
      Version.FULL_VERSION,
      StatsHelper.V1.toParquetStatistics(stats),
      binaryType);

  Assert.assertFalse("Stats should not include min/max: " + convertedStats, convertedStats.hasNonNullValue());
  Assert.assertTrue("Stats should have null count: " + convertedStats, convertedStats.isNumNullsSet());
  Assert.assertEquals("Stats should have 3 nulls: " + convertedStats, 3L, convertedStats.getNumNulls());
}
 
Example 5
Source Project: parquet-mr   Source File: TestTypeBuilders.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInt32AnnotationsRejectNonInt32() {
  OriginalType[] types = new OriginalType[] {
      DATE, TIME_MILLIS, UINT_8, UINT_16, UINT_32, INT_8, INT_16, INT_32};
  for (final OriginalType logicalType : types) {
    PrimitiveTypeName[] nonInt32 = new PrimitiveTypeName[]{
        BOOLEAN, INT64, INT96, DOUBLE, FLOAT, BINARY
    };
    for (final PrimitiveTypeName type : nonInt32) {
      assertThrows("Should reject non-int32 type: " + type,
          IllegalStateException.class, (Callable<Type>) () -> Types.required(type).as(logicalType).named("col"));
    }
    assertThrows("Should reject non-int32 type: FIXED_LEN_BYTE_ARRAY",
        IllegalStateException.class, (Callable<Type>) () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
            .as(logicalType).named("col"));
  }
}
 
Example 6
Source Project: parquet-mr   Source File: TestParquetMetadataConverter.java    License: Apache License 2.0 6 votes vote down vote up
private void testStillUseStatsWithSignedSortOrderIfSingleValue(StatsHelper helper) {
  ParquetMetadataConverter converter = new ParquetMetadataConverter();
  BinaryStatistics stats = new BinaryStatistics();
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();

  PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("b");
  Statistics convertedStats = converter.fromParquetStatistics(
      Version.FULL_VERSION,
      ParquetMetadataConverter.toParquetStatistics(stats),
      binaryType);

  Assert.assertFalse("Stats should not be empty: " + convertedStats, convertedStats.isEmpty());
  Assert.assertArrayEquals("min == max: " + convertedStats, convertedStats.getMaxBytes(), convertedStats.getMinBytes());
}
 
Example 7
Source Project: parquet-mr   Source File: TestParquetFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToThriftStatistics() throws Exception {
  long[] longArray = new long[] {39L, 99L, 12L, 1000L, 65L, 542L, 2533461316L, -253346131996L, Long.MAX_VALUE, Long.MIN_VALUE};
  LongStatistics parquetMRstats = new LongStatistics();

  for (long l: longArray) {
    parquetMRstats.updateStats(l);
  }
  final String createdBy =
      "parquet-mr version 1.8.0 (build d4d5a07ec9bd262ca1e93c309f1d7d4a74ebda4c)";
  Statistics thriftStats =
      org.apache.parquet.format.converter.ParquetMetadataConverter.toParquetStatistics(parquetMRstats);
  LongStatistics convertedBackStats =
      (LongStatistics) org.apache.parquet.format.converter.ParquetMetadataConverter.fromParquetStatistics(
          createdBy, thriftStats, PrimitiveTypeName.INT64);

  assertEquals(parquetMRstats.getMax(), convertedBackStats.getMax());
  assertEquals(parquetMRstats.getMin(), convertedBackStats.getMin());
  assertEquals(parquetMRstats.getNumNulls(), convertedBackStats.getNumNulls());
}
 
Example 8
@Test
public void testInt64AnnotationsRejectNonInt64() {
  LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
    timeType(true, MICROS), timeType(false, MICROS),
    timeType(true, NANOS), timeType(false, NANOS),
    timestampType(true, MILLIS), timestampType(false, MILLIS),
    timestampType(true, MICROS), timestampType(false, MICROS),
    timestampType(true, NANOS), timestampType(false, NANOS),
    intType(64, true), intType(64, false)};
  for (final LogicalTypeAnnotation logicalType : types) {
    PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[]{
        BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY
    };
    for (final PrimitiveTypeName type : nonInt64) {
      assertThrows("Should reject non-int64 type: " + type,
          IllegalStateException.class, (Callable<Type>) () -> Types.required(type).as(logicalType).named("col"));
    }
    assertThrows("Should reject non-int64 type: FIXED_LEN_BYTE_ARRAY",
        IllegalStateException.class, (Callable<Type>) () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
            .as(logicalType).named("col"));
  }
}
 
Example 9
Source Project: parquet-mr   Source File: ValidatingRecordConsumer.java    License: Apache License 2.0 6 votes vote down vote up
private void validate(PrimitiveTypeName p) {
  Type currentType = types.peek().asGroupType().getType(fields.peek());
  int c = fieldValueCount.pop() + 1;
  fieldValueCount.push(c);
  LOG.debug("validate {} for {}",p ,currentType.getName());
  switch (currentType.getRepetition()) {
    case OPTIONAL:
    case REQUIRED:
      if (c > 1) {
        throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
      }
      break;
    case REPEATED:
      break;
    default:
      throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
  }
  if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
    throw new InvalidRecordException("expected type " + p + " but got "+ currentType);
  }
}
 
Example 10
Source Project: parquet-mr   Source File: TestMemPageStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws IOException {
  MemPageStore memPageStore = new MemPageStore(10);
  ColumnDescriptor col = new ColumnDescriptor(path , PrimitiveTypeName.INT64, 2, 2);
  LongStatistics stats = new LongStatistics();
  PageWriter pageWriter = memPageStore.getPageWriter(col);
  pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  PageReader pageReader = memPageStore.getPageReader(col);
  long totalValueCount = pageReader.getTotalValueCount();
  System.out.println(totalValueCount);
  int total = 0;
  do {
    DataPage readPage = pageReader.readPage();
    total += readPage.getValueCount();
    System.out.println(readPage);
    // TODO: assert
  } while (total < totalValueCount);
}
 
Example 11
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInt32_V2_NoDict() {
  doTestValueWriter(
    PrimitiveTypeName.INT32,
    WriterVersion.PARQUET_2_0,
    false,
    false,
    DeltaBinaryPackingValuesWriter.class);
}
 
Example 12
Source Project: Bats   Source File: ParquetColumnMetadata.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns data type length for a given {@see ColumnDescriptor} and it's corresponding
 * {@see SchemaElement}. Neither is enough information alone as the max
 * repetition level (indicating if it is an array type) is in the ColumnDescriptor and
 * the length of a fixed width field is stored at the schema level.
 *
 * @return the length if fixed width, else <tt>UNDEFINED_LENGTH</tt> (-1)
 */
public int getDataTypeLength() {
  if (! isFixedLength()) {
    return UNDEFINED_LENGTH;
  } else if (isRepeated()) {
    return UNDEFINED_LENGTH;
  } else if (column.getType() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
    return se.getType_length() * 8;
  } else {
    return getTypeLengthInBits(column.getType());
  }
}
 
Example 13
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInt96_V2() {
  doTestValueWriter(
    PrimitiveTypeName.INT96,
    WriterVersion.PARQUET_2_0,
    true,
    false,
    PlainFixedLenArrayDictionaryValuesWriter.class, FixedLenByteArrayPlainValuesWriter.class);
}
 
Example 14
Source Project: Bats   Source File: ColumnReader.java    License: Apache License 2.0 5 votes vote down vote up
protected ColumnReader(ParquetRecordReader parentReader, ColumnDescriptor descriptor,
    ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
  this.parentReader = parentReader;
  this.columnDescriptor = descriptor;
  this.columnChunkMetaData = columnChunkMetaData;
  this.isFixedLength = fixedLength;
  this.schemaElement = schemaElement;
  this.valueVec =  v;
  boolean useAsyncPageReader = parentReader.useAsyncPageReader;
  if (useAsyncPageReader) {
    this.pageReader =
        new AsyncPageReader(this, parentReader.getFileSystem(), parentReader.getHadoopPath(),
            columnChunkMetaData);
  } else {
    this.pageReader =
        new PageReader(this, parentReader.getFileSystem(), parentReader.getHadoopPath(),
            columnChunkMetaData);
  }
  try {
    pageReader.init();
  } catch (IOException e) {
    UserException ex = UserException.dataReadError(e)
        .message("Error initializing page reader for Parquet file")
        .pushContext("Row Group Start: ", this.columnChunkMetaData.getStartingPos())
        .pushContext("Column: ", this.schemaElement.getName())
        .pushContext("File: ", this.parentReader.getHadoopPath().toString() )
        .build(logger);
    throw ex;
  }
  if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
    if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
    } else {
      dataTypeLengthInBits = ParquetColumnMetadata.getTypeLengthInBits(columnDescriptor.getType());
    }
  }
  threadPool = parentReader.getOperatorContext().getScanDecodeExecutor();
}
 
Example 15
Source Project: Bats   Source File: ParquetTypeHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static int getMaxPrecisionForPrimitiveType(PrimitiveTypeName type) {
  switch(type) {
    case INT32:
      return 9;
    case INT64:
      return 18;
    case FIXED_LEN_BYTE_ARRAY:
      return DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision();
    default:
      throw new UnsupportedOperationException(String.format(
        "Specified PrimitiveTypeName %s cannot be used to determine max precision",
        type));
  }
}
 
Example 16
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFixedLenByteArray() {
  doTestValueWriter(
    PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
    WriterVersion.PARQUET_1_0,
    true,
    false,
    FixedLenByteArrayPlainValuesWriter.class);
}
 
Example 17
Source Project: parquet-mr   Source File: ParquetMetadataConverter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param primitive a primitive physical type
 * @return the default sort order used when the logical type is not known
 */
private static SortOrder defaultSortOrder(PrimitiveTypeName primitive) {
  switch (primitive) {
    case BOOLEAN:
    case INT32:
    case INT64:
    case FLOAT:
    case DOUBLE:
      return SortOrder.SIGNED;
    case BINARY:
    case FIXED_LEN_BYTE_ARRAY:
      return SortOrder.UNSIGNED;
  }
  return SortOrder.UNKNOWN;
}
 
Example 18
Source Project: parquet-mr   Source File: TestValidTypeMap.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnsupportedType() {
  try {
    assertTypeValid(invalidColumn, PrimitiveTypeName.INT32);
    fail("This should throw!");
  } catch (IllegalArgumentException e) {
    assertEquals("Column invalid.column was declared as type: "
        + "org.apache.parquet.filter2.predicate.TestValidTypeMap$InvalidColumnType which is not supported "
        + "in FilterPredicates. Supported types for this column are: [class java.lang.Integer]", e.getMessage());
  }
}
 
Example 19
Source Project: dremio-oss   Source File: ParquetRecordWriter.java    License: Apache License 2.0 5 votes vote down vote up
private PrimitiveType getPrimitiveType(Field field) {
  MajorType majorType = getMajorTypeForField(field);
  MinorType minorType = majorType.getMinorType();
  String name = field.getName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (primitiveTypeName == null) {
    return null;
  }
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  DecimalMetadata decimalMetadata  = ParquetTypeHelper.getDecimalMetadataForField(majorType);
  return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
 
Example 20
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInt64() {
  doTestValueWriter(
    PrimitiveTypeName.INT64,
    WriterVersion.PARQUET_1_0,
    true,
    false,
    PlainLongDictionaryValuesWriter.class, PlainValuesWriter.class);
}
 
Example 21
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInt64_V2_NoDict() {
  doTestValueWriter(
    PrimitiveTypeName.INT64,
    WriterVersion.PARQUET_2_0,
    false,
    false,
    DeltaBinaryPackingValuesWriterForLong.class);
}
 
Example 22
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInt64_V2() {
  doTestValueWriter(
    PrimitiveTypeName.INT64,
    WriterVersion.PARQUET_2_0,
    true,
    false,
    PlainLongDictionaryValuesWriter.class, DeltaBinaryPackingValuesWriterForLong.class);
}
 
Example 23
/**
 * @param jsonSchema
 * @param repeated
 * @param outputType
 */
public PrimitiveConverter(JsonSchema jsonSchema, boolean repeated, PrimitiveTypeName outputType) {
  super(jsonSchema);
  this.repeated = repeated;
  this.outputType = outputType;
  this.schema = buildSchema();
}
 
Example 24
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, boolean enableByteStreamSplit, Class<? extends ValuesWriter> initialValueWriterClass, Class<? extends ValuesWriter> fallbackValueWriterClass) {
  ColumnDescriptor mockPath = createColumnDescriptor(typeName);
  ValuesWriterFactory factory = getDefaultFactory(version, enableDictionary, enableByteStreamSplit);
  ValuesWriter writer = factory.newValuesWriter(mockPath);

  validateFallbackWriter(writer, initialValueWriterClass, fallbackValueWriterClass);
}
 
Example 25
Source Project: parquet-mr   Source File: MetadataUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void showDetails(PrettyPrintWriter out, ColumnDescriptor desc) {
  String path = Joiner.on(".").skipNulls().join(desc.getPath());
  PrimitiveTypeName type = desc.getType();
  int defl = desc.getMaxDefinitionLevel();
  int repl = desc.getMaxRepetitionLevel();

  out.format("column desc: %s T:%s R:%d D:%d%n", path, type, repl, defl);
}
 
Example 26
Source Project: parquet-mr   Source File: MetadataUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath, boolean showOriginalTypes) {
  String name = Strings.repeat(".", depth) + type.getName();
  Repetition rep = type.getRepetition();
  PrimitiveTypeName ptype = type.getPrimitiveTypeName();

  out.format("%s: %s %s", name, rep, ptype);
  if (showOriginalTypes) {
    OriginalType otype;
    try {
      otype = type.getOriginalType();
    } catch (Exception e) {
      otype = null;
    }
    if (otype != null) out.format(" O:%s", otype);
  } else {
    LogicalTypeAnnotation ltype = type.getLogicalTypeAnnotation();
    if (ltype != null) out.format(" L:%s", ltype);
  }

  if (container != null) {
    cpath.add(type.getName());
    String[] paths = cpath.toArray(new String[0]);
    cpath.remove(cpath.size() - 1);

    ColumnDescriptor desc = container.getColumnDescription(paths);

    int defl = desc.getMaxDefinitionLevel();
    int repl = desc.getMaxRepetitionLevel();
    out.format(" R:%d D:%d", repl, defl);
  }
  out.println();
}
 
Example 27
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDouble_V2_WithByteStreamSplit() {
  doTestValueWriter(
    PrimitiveTypeName.DOUBLE,
    WriterVersion.PARQUET_2_0,
    false,
    true,
    ByteStreamSplitValuesWriter.class);
}
 
Example 28
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, boolean enableByteStreamSplit, Class<? extends ValuesWriter> expectedValueWriterClass) {
  ColumnDescriptor mockPath = createColumnDescriptor(typeName);
  ValuesWriterFactory factory = getDefaultFactory(version, enableDictionary, enableByteStreamSplit);
  ValuesWriter writer = factory.newValuesWriter(mockPath);

  validateWriterType(writer, expectedValueWriterClass);
}
 
Example 29
Source Project: parquet-mr   Source File: DefaultValuesWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDouble_V2_WithByteStreamSplitAndDictionary() {
  doTestValueWriter(
    PrimitiveTypeName.DOUBLE,
    WriterVersion.PARQUET_2_0,
    true,
    true,
    PlainDoubleDictionaryValuesWriter.class, ByteStreamSplitValuesWriter.class);
}
 
Example 30
Source Project: parquet-mr   Source File: ValidTypeMap.java    License: Apache License 2.0 5 votes vote down vote up
private static void add(Class<?> c, PrimitiveTypeName p) {
  Set<PrimitiveTypeName> descriptors = classToParquetType.get(c);
  if (descriptors == null) {
    descriptors = new HashSet<>();
    classToParquetType.put(c, descriptors);
  }
  descriptors.add(p);

  Set<Class<?>> classes = parquetTypeToClass.get(p);
  if (classes == null) {
    classes = new HashSet<>();
    parquetTypeToClass.put(p, classes);
  }
  classes.add(c);
}