org.apache.hadoop.hive.serde2.io.DateWritable Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.io.DateWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestEsriJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestEpochParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new EsriJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"attributes\":{\"when\":147147147147}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
	Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
						((DateWritable)fieldData).get().toString());
       value.set("{\"attributes\":{\"when\":142857142857}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
	Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
						((DateWritable)fieldData).get().toString());
}
 
Example #2
Source File: SerDeUtils.java    From presto with Apache License 2.0 6 votes vote down vote up
private static long formatDateAsLong(Object object, DateObjectInspector inspector)
{
    if (object instanceof LazyDate) {
        return ((LazyDate) object).getWritableObject().getDays();
    }
    if (object instanceof DateWritable) {
        return ((DateWritable) object).getDays();
    }

    // Hive will return java.sql.Date at midnight in JVM time zone
    long millisLocal = inspector.getPrimitiveJavaObject(object).getTime();
    // Convert it to midnight in UTC
    long millisUtc = DateTimeZone.getDefault().getMillisKeepLocal(DateTimeZone.UTC, millisLocal);
    // Convert midnight UTC to days
    return TimeUnit.MILLISECONDS.toDays(millisUtc);
}
 
Example #3
Source File: TestEsriJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new EsriJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"attributes\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example #4
Source File: TestGeoJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestEpochParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":147147147147}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
	Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":142857142857}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
	Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
						((DateWritable)fieldData).get().toString());
}
 
Example #5
Source File: TestGeoJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example #6
Source File: TestWritableTypeConverter.java    From tajo with Apache License 2.0 6 votes vote down vote up
@Test
public void testDateTypeConverting() throws UnsupportedDataTypeException {
  Datum testDatum;
  TimeMeta testTM = new TimeMeta();

  testTM.years = 1977;
  testTM.monthOfYear = 6;
  testTM.dayOfMonth = 28;

  testDatum = new DateDatum(testTM);

  Writable resultWritable = WritableTypeConverter.convertDatum2Writable(testDatum);
  assertEquals("1977-06-28", ((DateWritable)resultWritable).get().toString());

  Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable);
  assertEquals(testDatum, resultDatum);
}
 
Example #7
Source File: DateStatistics.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
public static ColumnStatistics getPartitionColumnStatistics(String value) throws IOException {
    DateStatistics dateStatistics = null;
    if(value != null) {
        Integer dateInDays = DateWritable.dateToDays(java.sql.Date.valueOf(value));
        dateStatistics = new DateStatistics(dateInDays, dateInDays);
    }
    return new ColumnStatistics(SpliceOrcNewInputFormat.DEFAULT_PARTITION_SIZE,
            null,null,null,null,dateStatistics,null,null);
}
 
Example #8
Source File: OrcFlowFileWriter.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        // Using the Writable here as it's used directly for writing as well as for stats.
        DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        indexStatistics.updateDate(val);
        writer.write(val.getDays());
        if (createBloomFilter) {
            bloomFilter.addLong(val.getDays());
        }
    }
}
 
Example #9
Source File: WritableTypeConverter.java    From tajo with Apache License 2.0 5 votes vote down vote up
public static Datum convertWritable2Datum(Writable value) throws UnsupportedDataTypeException {
  if (value == null) {
    return NullDatum.get();
  }

  DataType type = convertWritableToTajoType(value.getClass());

  switch(type.getType()) {
    case INT1: return new Int2Datum(((ByteWritable)value).get());
    case INT2: return new Int2Datum(((ShortWritable)value).get());
    case INT4: return new Int4Datum(((IntWritable)value).get());
    case INT8: return new Int8Datum(((LongWritable)value).get());

    case FLOAT4: return new Float4Datum(((FloatWritable)value).get());
    case FLOAT8: return new Float8Datum(((DoubleWritable)value).get());

    case DATE: return new DateDatum(((DateWritable)value).getDays() + DateTimeConstants.UNIX_EPOCH_JDATE);
    case TIMESTAMP: return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(
        ((TimestampWritable)value).getTimestamp().getTime()));

    case CHAR: return new CharDatum(value.toString());
    case TEXT: return new TextDatum(value.toString());
    case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes());
  }

  throw new TajoRuntimeException(new UnsupportedDataTypeException(value.getClass().getTypeName()));
}
 
Example #10
Source File: WritableTypeConverter.java    From tajo with Apache License 2.0 5 votes vote down vote up
public static Writable convertDatum2Writable(Datum value) {
  switch(value.kind()) {
    case INT1: return new ByteWritable(value.asByte());
    case INT2: return new ShortWritable(value.asInt2());
    case INT4: return new IntWritable(value.asInt4());
    case INT8: return new LongWritable(value.asInt8());

    case FLOAT4: return new FloatWritable(value.asFloat4());
    case FLOAT8: return new DoubleWritable(value.asFloat8());

    // NOTE: value should be DateDatum
    case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE);

    // NOTE: value should be TimestampDatum
    case TIMESTAMP:
      TimestampWritable result = new TimestampWritable();
      result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8()));
      return result;

    case CHAR: {
      String str = value.asChars();
      return new HiveCharWritable(new HiveChar(str, str.length()));
    }
    case TEXT: return new Text(value.asChars());
    case VARBINARY: return new BytesWritable(value.asByteArray());

    case NULL_TYPE: return null;
  }

  throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type())));
}
 
Example #11
Source File: HiveDateTimeTest.java    From indexr with Apache License 2.0 5 votes vote down vote up
@Test
public void testDateTime() {
    LocalDate date = LocalDate.now();
    Date sqlDate = Date.valueOf(date);
    long epochMillis = DateTimeUtil.getEpochMillisecond(sqlDate);
    Assert.assertEquals(date, DateTimeUtil.getLocalDate(epochMillis));
    Assert.assertEquals(sqlDate, DateTimeUtil.getJavaSQLDate(epochMillis));

    DateWritable dateWritable = new DateWritable(DateTimeUtil.getJavaSQLDate(epochMillis));
    Assert.assertEquals(epochMillis, DateTimeUtil.getEpochMillisecond(dateWritable.get()));
}
 
Example #12
Source File: ObjectInspectors.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
public DateWritable getPrimitiveWritableObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableDateMilliHolder h = (NullableDateMilliHolder) o;
<#else>
  final DateMilliHolder h = (DateMilliHolder) o;
</#if>
  return new DateWritable(new java.sql.Date(h.value));
}
 
Example #13
Source File: OrcFlowFileWriter.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        // Using the Writable here as it's used directly for writing as well as for stats.
        DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        indexStatistics.updateDate(val);
        writer.write(val.getDays());
        if (createBloomFilter) {
            bloomFilter.addLong(val.getDays());
        }
    }
}
 
Example #14
Source File: WritableTypeConverter.java    From tajo with Apache License 2.0 4 votes vote down vote up
public static DataType convertWritableToTajoType(Class<? extends Writable> writableClass) throws UnsupportedDataTypeException {
  if (writableClass == null)
    return null;

  Set<Class<?>> parents = ReflectionUtils.getAllSuperTypes(writableClass);

  if (writableClass == ByteWritable.class || parents.contains(ByteWritable.class)) {
    return builder.setType(Type.INT1).build();
  }
  if (writableClass == ShortWritable.class || parents.contains(ShortWritable.class)) {
    return builder.setType(Type.INT2).build();
  }
  if (writableClass == IntWritable.class || parents.contains(IntWritable.class)) {
    return builder.setType(Type.INT4).build();
  }
  if (writableClass == LongWritable.class || parents.contains(LongWritable.class)) {
    return builder.setType(Type.INT8).build();
  }
  if (writableClass == HiveCharWritable.class || parents.contains(HiveCharWritable.class)) {
    return builder.setType(Type.CHAR).build();
  }
  if (writableClass == Text.class || parents.contains(Text.class)) {
    return builder.setType(Type.TEXT).build();
  }
  if (writableClass == FloatWritable.class || parents.contains(FloatWritable.class)) {
    return builder.setType(Type.FLOAT4).build();
  }
  if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) {
    return builder.setType(Type.FLOAT8).build();
  }
  if (writableClass == DateWritable.class || parents.contains(DateWritable.class)) {
    return builder.setType(Type.DATE).build();
  }
  if (writableClass == TimestampWritable.class || parents.contains(TimestampWritable.class)) {
    return builder.setType(Type.TIMESTAMP).build();
  }
  if (writableClass == BytesWritable.class || parents.contains(BytesWritable.class)) {
    return builder.setType(Type.VARBINARY).build();
  }

  throw new UnsupportedDataTypeException(writableClass.getSimpleName());
}
 
Example #15
Source File: IndexRRecordWriter.java    From indexr with Apache License 2.0 4 votes vote down vote up
@Override
public void write(Writable w) throws IOException {
    ArrayWritable datas = (ArrayWritable) w;
    for (int colId = 0; colId < sqlTypes.length; colId++) {
        SQLType type = sqlTypes[colId];
        Writable currentValue = datas.get()[colId];
        switch (type) {
            case INT:
                if (currentValue == null) {
                    rowBuilder.appendInt(0);
                } else {
                    rowBuilder.appendInt(((IntWritable) currentValue).get());
                }
                break;
            case BIGINT:
                if (currentValue == null) {
                    rowBuilder.appendLong(0L);
                } else {
                    rowBuilder.appendLong(((LongWritable) currentValue).get());
                }
                break;
            case FLOAT:
                if (currentValue == null) {
                    rowBuilder.appendFloat(0f);
                } else {
                    rowBuilder.appendFloat(((FloatWritable) currentValue).get());
                }
                break;
            case DOUBLE:
                if (currentValue == null) {
                    rowBuilder.appendDouble(0d);
                } else {
                    rowBuilder.appendDouble(((DoubleWritable) currentValue).get());
                }
                break;
            case VARCHAR:
                if (currentValue == null) {
                    rowBuilder.appendString("");
                } else {
                    Text v = (Text) currentValue;
                    rowBuilder.appendUTF8String(v.getBytes(), 0, v.getLength());
                }
                break;
            case DATE:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((DateWritable) currentValue).get()));
                }
                break;
            case DATETIME:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((TimestampWritable) currentValue).getTimestamp()));
                }
                break;
            default:
                throw new IOException("can't recognize this type [" + type + "]");
        }
    }
    segmentGen.add(rowBuilder.buildAndReset());
}
 
Example #16
Source File: OrcUtils.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
/**
 * Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
 */
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
  if (from instanceof ByteWritable) {
    if (to instanceof ByteWritable) {
      ((ByteWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ByteWritable) from).get());
      return;
    }
  } else if (from instanceof ShortWritable) {
    if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ShortWritable) from).get());
      return;
    }
  } else if (from instanceof IntWritable) {
    if (to instanceof IntWritable) {
      ((IntWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((IntWritable) from).get());
      return;
    }
  } else if (from instanceof LongWritable) {
    if (to instanceof LongWritable) {
      ((LongWritable) to).set(((LongWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((LongWritable) from).get());
      return;
    }
    // Following from this branch, type-widening is not allowed and only value-copy will happen.
  } else if (from instanceof DoubleWritable) {
    if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((DoubleWritable) from).get());
      return;
    }
  } else if (from instanceof BytesWritable) {
    if (to instanceof BytesWritable) {
      ((BytesWritable) to).set((BytesWritable) from);
      return;
    }
  } else if (from instanceof FloatWritable) {
    if (to instanceof FloatWritable) {
      ((FloatWritable) to).set(((FloatWritable) from).get());
      return;
    }
  } else if (from instanceof Text) {
    if (to instanceof Text) {
      ((Text) to).set((Text) from);
      return;
    }
  } else if (from instanceof DateWritable) {
    if (to instanceof DateWritable) {
      ((DateWritable) to).set(((DateWritable) from).get());
      return;
    }
  } else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
    ((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
    return;
  } else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
    ((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
    return;
  } else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
    ((BooleanWritable) to).set(((BooleanWritable) from).get());
    return;
  }
  throw new UnsupportedOperationException(String
      .format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
          from.getClass(), to.getClass()));
}
 
Example #17
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 4 votes vote down vote up
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            setUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // To prevent similar mistakes like the one described in https://github.com/pinterest/secor/pull/1018,
            // it would be better to explicitly throw an exception here rather than ignore the incoming values,
            // which causes silent failures in a later stage.
            throw new UnsupportedOperationException();
        case MAP:
            setMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}
 
Example #18
Source File: HiveFieldConverter.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final DateWritable writeable = ((DateObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  ((DateMilliVector) outputVV).setSafe(outputIndex, writeable.get().toLocalDate().toEpochDay() * MILLIS_PER_DAY);
}
 
Example #19
Source File: BaseJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Copies the Writable at fieldIndex from rowBase to row, then sets the value of the Writable
 * to the value in parser
 * 
 * @param fieldIndex column index of field in row
 * @param parser JsonParser pointing to the attribute
 * @throws JsonParseException
 * @throws IOException
 */
private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{

	PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);
	if (JsonToken.VALUE_NULL == parser.getCurrentToken())
		return;  // leave the row-cell as null

	// set the field in the row to the writable from rowBase
	row.set(fieldIndex, rowBase.get(fieldIndex));

	switch (poi.getPrimitiveCategory()){
	case BYTE:
		((ByteWritable)row.get(fieldIndex)).set(parser.getByteValue());
		break;
	case SHORT:
		((ShortWritable)row.get(fieldIndex)).set(parser.getShortValue());
		break;
	case INT:
		((IntWritable)row.get(fieldIndex)).set(parser.getIntValue());
		break;
	case LONG:
		((LongWritable)row.get(fieldIndex)).set(parser.getLongValue());
		break;
	case DOUBLE:
		((DoubleWritable)row.get(fieldIndex)).set(parser.getDoubleValue());
		break;
	case FLOAT:
		((FloatWritable)row.get(fieldIndex)).set(parser.getFloatValue());
		break;
	case BOOLEAN:
		((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
		break;
	case DATE:    // DateWritable stores days not milliseconds.
		((DateWritable)row.get(fieldIndex)).set(parseDate(parser));
		break;
	case TIMESTAMP:
		((TimestampWritable)row.get(fieldIndex)).set(parseTime(parser));
		break;
	default:    // STRING/unrecognized
		((Text)row.get(fieldIndex)).set(parser.getText());
		break;	
	}
}
 
Example #20
Source File: HiveInspectors.java    From flink with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getObjectInspector(Class clazz) {
	TypeInfo typeInfo;

	if (clazz.equals(String.class) || clazz.equals(Text.class)) {

		typeInfo = TypeInfoFactory.stringTypeInfo;
	} else if (clazz.equals(Boolean.class) || clazz.equals(BooleanWritable.class)) {

		typeInfo = TypeInfoFactory.booleanTypeInfo;
	} else if (clazz.equals(Byte.class) || clazz.equals(ByteWritable.class)) {

		typeInfo = TypeInfoFactory.byteTypeInfo;
	} else if (clazz.equals(Short.class) || clazz.equals(ShortWritable.class)) {

		typeInfo = TypeInfoFactory.shortTypeInfo;
	} else if (clazz.equals(Integer.class) || clazz.equals(IntWritable.class)) {

		typeInfo = TypeInfoFactory.intTypeInfo;
	} else if (clazz.equals(Long.class) || clazz.equals(LongWritable.class)) {

		typeInfo = TypeInfoFactory.longTypeInfo;
	} else if (clazz.equals(Float.class) || clazz.equals(FloatWritable.class)) {

		typeInfo = TypeInfoFactory.floatTypeInfo;
	} else if (clazz.equals(Double.class) || clazz.equals(DoubleWritable.class)) {

		typeInfo = TypeInfoFactory.doubleTypeInfo;
	} else if (clazz.equals(Date.class) || clazz.equals(DateWritable.class)) {

		typeInfo = TypeInfoFactory.dateTypeInfo;
	} else if (clazz.equals(Timestamp.class) || clazz.equals(TimestampWritable.class)) {

		typeInfo = TypeInfoFactory.timestampTypeInfo;
	} else if (clazz.equals(byte[].class) || clazz.equals(BytesWritable.class)) {

		typeInfo = TypeInfoFactory.binaryTypeInfo;
	} else if (clazz.equals(HiveChar.class) || clazz.equals(HiveCharWritable.class)) {

		typeInfo = TypeInfoFactory.charTypeInfo;
	} else if (clazz.equals(HiveVarchar.class) || clazz.equals(HiveVarcharWritable.class)) {

		typeInfo = TypeInfoFactory.varcharTypeInfo;
	} else if (clazz.equals(HiveDecimal.class) || clazz.equals(HiveDecimalWritable.class)) {

		typeInfo = TypeInfoFactory.decimalTypeInfo;
	} else {
		throw new FlinkHiveUDFException(
			String.format("Class %s is not supported yet", clazz.getName()));
	}

	return getObjectInspector(typeInfo);
}
 
Example #21
Source File: JsonSerDeTestingBase.java    From spatial-framework-for-hadoop with Apache License 2.0 4 votes vote down vote up
protected void addWritable(ArrayList<Object> stuff, java.sql.Date item) {
	stuff.add(new DateWritable(item));
}
 
Example #22
Source File: TestDataWritableWriter.java    From presto with Apache License 2.0 4 votes vote down vote up
/**
 * It writes the primitive value to the Parquet RecordConsumer.
 *
 * @param value The object that contains the primitive value.
 * @param inspector The object inspector used to get the correct value type.
 */
private void writePrimitive(Object value, PrimitiveObjectInspector inspector)
{
    if (value == null) {
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case VOID:
            return;
        case DOUBLE:
            recordConsumer.addDouble(((DoubleObjectInspector) inspector).get(value));
            break;
        case BOOLEAN:
            recordConsumer.addBoolean(((BooleanObjectInspector) inspector).get(value));
            break;
        case FLOAT:
            recordConsumer.addFloat(((FloatObjectInspector) inspector).get(value));
            break;
        case BYTE:
            recordConsumer.addInteger(((ByteObjectInspector) inspector).get(value));
            break;
        case INT:
            recordConsumer.addInteger(((IntObjectInspector) inspector).get(value));
            break;
        case LONG:
            recordConsumer.addLong(((LongObjectInspector) inspector).get(value));
            break;
        case SHORT:
            recordConsumer.addInteger(((ShortObjectInspector) inspector).get(value));
            break;
        case STRING:
            String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromString(v));
            break;
        case CHAR:
            String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
            recordConsumer.addBinary(Binary.fromString(vChar));
            break;
        case VARCHAR:
            String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
            recordConsumer.addBinary(Binary.fromString(vVarchar));
            break;
        case BINARY:
            byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromByteArray(vBinary));
            break;
        case TIMESTAMP:
            Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary());
            break;
        case DECIMAL:
            HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value));
            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
            recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo));
            break;
        case DATE:
            Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addInteger(DateWritable.dateToDays(vDate));
            break;
        default:
            throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());
    }
}
 
Example #23
Source File: RcFileTester.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        if (SESSION.isLegacyTimestamp()) {
            actualValue = SqlTimestamp.legacyFromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
        }
        else {
            actualValue = SqlTimestamp.fromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L));
        }
    }
    else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    }
    else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    }
    else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}
 
Example #24
Source File: HiveWritableValueWriter.java    From elasticsearch-hadoop with Apache License 2.0 4 votes vote down vote up
static String toES(Object dateWritable) {
    DateWritable dw = (DateWritable) dateWritable;
    Calendar cal = Calendar.getInstance();
    cal.setTimeInMillis(dw.get().getTime());
    return DatatypeConverter.printDate(cal);
}
 
Example #25
Source File: DateColumnBlock.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
@Override
public void setPartitionValue(String value, int size) {
    columnVector = ColumnVector.allocate(size, dataType, MemoryMode.ON_HEAP);
    columnVector.appendInts(size, DateWritable.dateToDays(java.sql.Date.valueOf(value)));
}
 
Example #26
Source File: OrcTester.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    }
    else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}