Java Code Examples for org.apache.hadoop.hive.serde2.io.DateWritable

The following examples show how to use org.apache.hadoop.hive.serde2.io.DateWritable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static long formatDateAsLong(Object object, DateObjectInspector inspector)
{
    if (object instanceof LazyDate) {
        return ((LazyDate) object).getWritableObject().getDays();
    }
    if (object instanceof DateWritable) {
        return ((DateWritable) object).getDays();
    }

    // Hive will return java.sql.Date at midnight in JVM time zone
    long millisLocal = inspector.getPrimitiveJavaObject(object).getTime();
    // Convert it to midnight in UTC
    long millisUtc = DateTimeZone.getDefault().getMillisKeepLocal(DateTimeZone.UTC, millisLocal);
    // Convert midnight UTC to days
    return TimeUnit.MILLISECONDS.toDays(millisUtc);
}
 
Example 2
Source Project: tajo   Source File: TestWritableTypeConverter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDateTypeConverting() throws UnsupportedDataTypeException {
  Datum testDatum;
  TimeMeta testTM = new TimeMeta();

  testTM.years = 1977;
  testTM.monthOfYear = 6;
  testTM.dayOfMonth = 28;

  testDatum = new DateDatum(testTM);

  Writable resultWritable = WritableTypeConverter.convertDatum2Writable(testDatum);
  assertEquals("1977-06-28", ((DateWritable)resultWritable).get().toString());

  Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable);
  assertEquals(testDatum, resultDatum);
}
 
Example 3
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example 4
@Test
public void TestEpochParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":147147147147}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
	Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":142857142857}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
	Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
						((DateWritable)fieldData).get().toString());
}
 
Example 5
@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new EsriJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"attributes\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 
Example 6
@Test
public void TestEpochParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new EsriJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"attributes\":{\"when\":147147147147}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
	Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
						((DateWritable)fieldData).get().toString());
       value.set("{\"attributes\":{\"when\":142857142857}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
	Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
						((DateWritable)fieldData).get().toString());
}
 
Example 7
Source Project: localization_nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        // Using the Writable here as it's used directly for writing as well as for stats.
        DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        indexStatistics.updateDate(val);
        writer.write(val.getDays());
        if (createBloomFilter) {
            bloomFilter.addLong(val.getDays());
        }
    }
}
 
Example 8
Source Project: dremio-oss   Source File: ObjectInspectors.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DateWritable getPrimitiveWritableObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableDateMilliHolder h = (NullableDateMilliHolder) o;
<#else>
  final DateMilliHolder h = (DateMilliHolder) o;
</#if>
  return new DateWritable(new java.sql.Date(h.value));
}
 
Example 9
Source Project: indexr   Source File: HiveDateTimeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDateTime() {
    LocalDate date = LocalDate.now();
    Date sqlDate = Date.valueOf(date);
    long epochMillis = DateTimeUtil.getEpochMillisecond(sqlDate);
    Assert.assertEquals(date, DateTimeUtil.getLocalDate(epochMillis));
    Assert.assertEquals(sqlDate, DateTimeUtil.getJavaSQLDate(epochMillis));

    DateWritable dateWritable = new DateWritable(DateTimeUtil.getJavaSQLDate(epochMillis));
    Assert.assertEquals(epochMillis, DateTimeUtil.getEpochMillisecond(dateWritable.get()));
}
 
Example 10
Source Project: tajo   Source File: WritableTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
public static Writable convertDatum2Writable(Datum value) {
  switch(value.kind()) {
    case INT1: return new ByteWritable(value.asByte());
    case INT2: return new ShortWritable(value.asInt2());
    case INT4: return new IntWritable(value.asInt4());
    case INT8: return new LongWritable(value.asInt8());

    case FLOAT4: return new FloatWritable(value.asFloat4());
    case FLOAT8: return new DoubleWritable(value.asFloat8());

    // NOTE: value should be DateDatum
    case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE);

    // NOTE: value should be TimestampDatum
    case TIMESTAMP:
      TimestampWritable result = new TimestampWritable();
      result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8()));
      return result;

    case CHAR: {
      String str = value.asChars();
      return new HiveCharWritable(new HiveChar(str, str.length()));
    }
    case TEXT: return new Text(value.asChars());
    case VARBINARY: return new BytesWritable(value.asByteArray());

    case NULL_TYPE: return null;
  }

  throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type())));
}
 
Example 11
Source Project: tajo   Source File: WritableTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
public static Datum convertWritable2Datum(Writable value) throws UnsupportedDataTypeException {
  if (value == null) {
    return NullDatum.get();
  }

  DataType type = convertWritableToTajoType(value.getClass());

  switch(type.getType()) {
    case INT1: return new Int2Datum(((ByteWritable)value).get());
    case INT2: return new Int2Datum(((ShortWritable)value).get());
    case INT4: return new Int4Datum(((IntWritable)value).get());
    case INT8: return new Int8Datum(((LongWritable)value).get());

    case FLOAT4: return new Float4Datum(((FloatWritable)value).get());
    case FLOAT8: return new Float8Datum(((DoubleWritable)value).get());

    case DATE: return new DateDatum(((DateWritable)value).getDays() + DateTimeConstants.UNIX_EPOCH_JDATE);
    case TIMESTAMP: return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(
        ((TimestampWritable)value).getTimestamp().getTime()));

    case CHAR: return new CharDatum(value.toString());
    case TEXT: return new TextDatum(value.toString());
    case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes());
  }

  throw new TajoRuntimeException(new UnsupportedDataTypeException(value.getClass().getTypeName()));
}
 
Example 12
Source Project: nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        // Using the Writable here as it's used directly for writing as well as for stats.
        DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        indexStatistics.updateDate(val);
        writer.write(val.getDays());
        if (createBloomFilter) {
            bloomFilter.addLong(val.getDays());
        }
    }
}
 
Example 13
public static ColumnStatistics getPartitionColumnStatistics(String value) throws IOException {
    DateStatistics dateStatistics = null;
    if(value != null) {
        Integer dateInDays = DateWritable.dateToDays(java.sql.Date.valueOf(value));
        dateStatistics = new DateStatistics(dateInDays, dateInDays);
    }
    return new ColumnStatistics(SpliceOrcNewInputFormat.DEFAULT_PARTITION_SIZE,
            null,null,null,null,dateStatistics,null,null);
}
 
Example 14
Source Project: presto   Source File: OrcTester.java    License: Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    }
    else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}
 
Example 15
Source Project: presto   Source File: RcFileTester.java    License: Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        if (SESSION.isLegacyTimestamp()) {
            actualValue = SqlTimestamp.legacyFromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
        }
        else {
            actualValue = SqlTimestamp.fromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L));
        }
    }
    else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    }
    else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    }
    else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}
 
Example 16
Source Project: presto   Source File: TestDataWritableWriter.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * It writes the primitive value to the Parquet RecordConsumer.
 *
 * @param value The object that contains the primitive value.
 * @param inspector The object inspector used to get the correct value type.
 */
private void writePrimitive(Object value, PrimitiveObjectInspector inspector)
{
    if (value == null) {
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case VOID:
            return;
        case DOUBLE:
            recordConsumer.addDouble(((DoubleObjectInspector) inspector).get(value));
            break;
        case BOOLEAN:
            recordConsumer.addBoolean(((BooleanObjectInspector) inspector).get(value));
            break;
        case FLOAT:
            recordConsumer.addFloat(((FloatObjectInspector) inspector).get(value));
            break;
        case BYTE:
            recordConsumer.addInteger(((ByteObjectInspector) inspector).get(value));
            break;
        case INT:
            recordConsumer.addInteger(((IntObjectInspector) inspector).get(value));
            break;
        case LONG:
            recordConsumer.addLong(((LongObjectInspector) inspector).get(value));
            break;
        case SHORT:
            recordConsumer.addInteger(((ShortObjectInspector) inspector).get(value));
            break;
        case STRING:
            String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromString(v));
            break;
        case CHAR:
            String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
            recordConsumer.addBinary(Binary.fromString(vChar));
            break;
        case VARCHAR:
            String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
            recordConsumer.addBinary(Binary.fromString(vVarchar));
            break;
        case BINARY:
            byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromByteArray(vBinary));
            break;
        case TIMESTAMP:
            Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary());
            break;
        case DECIMAL:
            HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value));
            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
            recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo));
            break;
        case DATE:
            Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addInteger(DateWritable.dateToDays(vDate));
            break;
        default:
            throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());
    }
}
 
Example 17
Source Project: flink   Source File: HiveInspectors.java    License: Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getObjectInspector(Class clazz) {
	TypeInfo typeInfo;

	if (clazz.equals(String.class) || clazz.equals(Text.class)) {

		typeInfo = TypeInfoFactory.stringTypeInfo;
	} else if (clazz.equals(Boolean.class) || clazz.equals(BooleanWritable.class)) {

		typeInfo = TypeInfoFactory.booleanTypeInfo;
	} else if (clazz.equals(Byte.class) || clazz.equals(ByteWritable.class)) {

		typeInfo = TypeInfoFactory.byteTypeInfo;
	} else if (clazz.equals(Short.class) || clazz.equals(ShortWritable.class)) {

		typeInfo = TypeInfoFactory.shortTypeInfo;
	} else if (clazz.equals(Integer.class) || clazz.equals(IntWritable.class)) {

		typeInfo = TypeInfoFactory.intTypeInfo;
	} else if (clazz.equals(Long.class) || clazz.equals(LongWritable.class)) {

		typeInfo = TypeInfoFactory.longTypeInfo;
	} else if (clazz.equals(Float.class) || clazz.equals(FloatWritable.class)) {

		typeInfo = TypeInfoFactory.floatTypeInfo;
	} else if (clazz.equals(Double.class) || clazz.equals(DoubleWritable.class)) {

		typeInfo = TypeInfoFactory.doubleTypeInfo;
	} else if (clazz.equals(Date.class) || clazz.equals(DateWritable.class)) {

		typeInfo = TypeInfoFactory.dateTypeInfo;
	} else if (clazz.equals(Timestamp.class) || clazz.equals(TimestampWritable.class)) {

		typeInfo = TypeInfoFactory.timestampTypeInfo;
	} else if (clazz.equals(byte[].class) || clazz.equals(BytesWritable.class)) {

		typeInfo = TypeInfoFactory.binaryTypeInfo;
	} else if (clazz.equals(HiveChar.class) || clazz.equals(HiveCharWritable.class)) {

		typeInfo = TypeInfoFactory.charTypeInfo;
	} else if (clazz.equals(HiveVarchar.class) || clazz.equals(HiveVarcharWritable.class)) {

		typeInfo = TypeInfoFactory.varcharTypeInfo;
	} else if (clazz.equals(HiveDecimal.class) || clazz.equals(HiveDecimalWritable.class)) {

		typeInfo = TypeInfoFactory.decimalTypeInfo;
	} else {
		throw new FlinkHiveUDFException(
			String.format("Class %s is not supported yet", clazz.getName()));
	}

	return getObjectInspector(typeInfo);
}
 
Example 18
Source Project: dremio-oss   Source File: HiveFieldConverter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final DateWritable writeable = ((DateObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  ((DateMilliVector) outputVV).setSafe(outputIndex, writeable.get().toLocalDate().toEpochDay() * MILLIS_PER_DAY);
}
 
Example 19
Source Project: indexr   Source File: IndexRRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean next(Void aVoid, SchemaWritable writable) throws IOException {
    if (segment == null) {
        return false;
    }
    if (offset >= segment.rowCount()) {
        return false;
    }

    Row current = rowIterator.next();
    Writable[] writables = new Writable[projectCols.length];
    for (int i = 0; i < projectCols.length; i++) {
        ColumnSchema columnSchema = projectCols[i];
        int colId = projectColIds[i];
        if (colId == -1 || columnSchema == null) {
            writables[i] = null;
            continue;
        }
        switch (columnSchema.getSqlType()) {
            case INT:
                writables[i] = new IntWritable(current.getInt(colId));
                break;
            case BIGINT:
                writables[i] = new LongWritable(current.getLong(colId));
                break;
            case FLOAT:
                writables[i] = new FloatWritable(current.getFloat(colId));
                break;
            case DOUBLE:
                writables[i] = new DoubleWritable(current.getDouble(colId));
                break;
            case VARCHAR:
                writables[i] = new Text(current.getString(colId).getBytes());
                //writables[i] =  new BytesWritable();
                break;
            case DATE:
                writables[i] = new DateWritable(DateTimeUtil.getJavaSQLDate(current.getLong(colId)));
                break;
            case DATETIME:
                writables[i] = new TimestampWritable(DateTimeUtil.getJavaSQLTimeStamp(current.getLong(colId)));
                break;
            default:
                throw new IllegalStateException("Illegal type: " + columnSchema.getSqlType());
        }
    }

    offset++;

    // Must set it every fucking time, as writable could be reused between segments!
    writable.columns = projectCols;
    writable.set(writables);

    return true;
}
 
Example 20
Source Project: indexr   Source File: IndexRRecordWriter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void write(Writable w) throws IOException {
    ArrayWritable datas = (ArrayWritable) w;
    for (int colId = 0; colId < sqlTypes.length; colId++) {
        SQLType type = sqlTypes[colId];
        Writable currentValue = datas.get()[colId];
        switch (type) {
            case INT:
                if (currentValue == null) {
                    rowBuilder.appendInt(0);
                } else {
                    rowBuilder.appendInt(((IntWritable) currentValue).get());
                }
                break;
            case BIGINT:
                if (currentValue == null) {
                    rowBuilder.appendLong(0L);
                } else {
                    rowBuilder.appendLong(((LongWritable) currentValue).get());
                }
                break;
            case FLOAT:
                if (currentValue == null) {
                    rowBuilder.appendFloat(0f);
                } else {
                    rowBuilder.appendFloat(((FloatWritable) currentValue).get());
                }
                break;
            case DOUBLE:
                if (currentValue == null) {
                    rowBuilder.appendDouble(0d);
                } else {
                    rowBuilder.appendDouble(((DoubleWritable) currentValue).get());
                }
                break;
            case VARCHAR:
                if (currentValue == null) {
                    rowBuilder.appendString("");
                } else {
                    Text v = (Text) currentValue;
                    rowBuilder.appendUTF8String(v.getBytes(), 0, v.getLength());
                }
                break;
            case DATE:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((DateWritable) currentValue).get()));
                }
                break;
            case DATETIME:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((TimestampWritable) currentValue).getTimestamp()));
                }
                break;
            default:
                throw new IOException("can't recognize this type [" + type + "]");
        }
    }
    segmentGen.add(rowBuilder.buildAndReset());
}
 
Example 21
Source Project: tajo   Source File: WritableTypeConverter.java    License: Apache License 2.0 4 votes vote down vote up
public static DataType convertWritableToTajoType(Class<? extends Writable> writableClass) throws UnsupportedDataTypeException {
  if (writableClass == null)
    return null;

  Set<Class<?>> parents = ReflectionUtils.getAllSuperTypes(writableClass);

  if (writableClass == ByteWritable.class || parents.contains(ByteWritable.class)) {
    return builder.setType(Type.INT1).build();
  }
  if (writableClass == ShortWritable.class || parents.contains(ShortWritable.class)) {
    return builder.setType(Type.INT2).build();
  }
  if (writableClass == IntWritable.class || parents.contains(IntWritable.class)) {
    return builder.setType(Type.INT4).build();
  }
  if (writableClass == LongWritable.class || parents.contains(LongWritable.class)) {
    return builder.setType(Type.INT8).build();
  }
  if (writableClass == HiveCharWritable.class || parents.contains(HiveCharWritable.class)) {
    return builder.setType(Type.CHAR).build();
  }
  if (writableClass == Text.class || parents.contains(Text.class)) {
    return builder.setType(Type.TEXT).build();
  }
  if (writableClass == FloatWritable.class || parents.contains(FloatWritable.class)) {
    return builder.setType(Type.FLOAT4).build();
  }
  if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) {
    return builder.setType(Type.FLOAT8).build();
  }
  if (writableClass == DateWritable.class || parents.contains(DateWritable.class)) {
    return builder.setType(Type.DATE).build();
  }
  if (writableClass == TimestampWritable.class || parents.contains(TimestampWritable.class)) {
    return builder.setType(Type.TIMESTAMP).build();
  }
  if (writableClass == BytesWritable.class || parents.contains(BytesWritable.class)) {
    return builder.setType(Type.VARBINARY).build();
  }

  throw new UnsupportedDataTypeException(writableClass.getSimpleName());
}
 
Example 22
Source Project: incubator-gobblin   Source File: OrcUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
 */
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
  if (from instanceof ByteWritable) {
    if (to instanceof ByteWritable) {
      ((ByteWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ByteWritable) from).get());
      return;
    }
  } else if (from instanceof ShortWritable) {
    if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ShortWritable) from).get());
      return;
    }
  } else if (from instanceof IntWritable) {
    if (to instanceof IntWritable) {
      ((IntWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((IntWritable) from).get());
      return;
    }
  } else if (from instanceof LongWritable) {
    if (to instanceof LongWritable) {
      ((LongWritable) to).set(((LongWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((LongWritable) from).get());
      return;
    }
    // Following from this branch, type-widening is not allowed and only value-copy will happen.
  } else if (from instanceof DoubleWritable) {
    if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((DoubleWritable) from).get());
      return;
    }
  } else if (from instanceof BytesWritable) {
    if (to instanceof BytesWritable) {
      ((BytesWritable) to).set((BytesWritable) from);
      return;
    }
  } else if (from instanceof FloatWritable) {
    if (to instanceof FloatWritable) {
      ((FloatWritable) to).set(((FloatWritable) from).get());
      return;
    }
  } else if (from instanceof Text) {
    if (to instanceof Text) {
      ((Text) to).set((Text) from);
      return;
    }
  } else if (from instanceof DateWritable) {
    if (to instanceof DateWritable) {
      ((DateWritable) to).set(((DateWritable) from).get());
      return;
    }
  } else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
    ((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
    return;
  } else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
    ((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
    return;
  } else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
    ((BooleanWritable) to).set(((BooleanWritable) from).get());
    return;
  }
  throw new UnsupportedOperationException(String
      .format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
          from.getClass(), to.getClass()));
}
 
Example 23
Source Project: secor   Source File: JsonFieldFiller.java    License: Apache License 2.0 4 votes vote down vote up
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            setUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // To prevent similar mistakes like the one described in https://github.com/pinterest/secor/pull/1018,
            // it would be better to explicitly throw an exception here rather than ignore the incoming values,
            // which causes silent failures in a later stage.
            throw new UnsupportedOperationException();
        case MAP:
            setMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}
 
Example 24
/**
 * Copies the Writable at fieldIndex from rowBase to row, then sets the value of the Writable
 * to the value in parser
 * 
 * @param fieldIndex column index of field in row
 * @param parser JsonParser pointing to the attribute
 * @throws JsonParseException
 * @throws IOException
 */
private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{

	PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);
	if (JsonToken.VALUE_NULL == parser.getCurrentToken())
		return;  // leave the row-cell as null

	// set the field in the row to the writable from rowBase
	row.set(fieldIndex, rowBase.get(fieldIndex));

	switch (poi.getPrimitiveCategory()){
	case BYTE:
		((ByteWritable)row.get(fieldIndex)).set(parser.getByteValue());
		break;
	case SHORT:
		((ShortWritable)row.get(fieldIndex)).set(parser.getShortValue());
		break;
	case INT:
		((IntWritable)row.get(fieldIndex)).set(parser.getIntValue());
		break;
	case LONG:
		((LongWritable)row.get(fieldIndex)).set(parser.getLongValue());
		break;
	case DOUBLE:
		((DoubleWritable)row.get(fieldIndex)).set(parser.getDoubleValue());
		break;
	case FLOAT:
		((FloatWritable)row.get(fieldIndex)).set(parser.getFloatValue());
		break;
	case BOOLEAN:
		((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
		break;
	case DATE:    // DateWritable stores days not milliseconds.
		((DateWritable)row.get(fieldIndex)).set(parseDate(parser));
		break;
	case TIMESTAMP:
		((TimestampWritable)row.get(fieldIndex)).set(parseTime(parser));
		break;
	default:    // STRING/unrecognized
		((Text)row.get(fieldIndex)).set(parser.getText());
		break;	
	}
}
 
Example 25
protected void addWritable(ArrayList<Object> stuff, java.sql.Date item) {
	stuff.add(new DateWritable(item));
}
 
Example 26
static String toES(Object dateWritable) {
    DateWritable dw = (DateWritable) dateWritable;
    Calendar cal = Calendar.getInstance();
    cal.setTimeInMillis(dw.get().getTime());
    return DatatypeConverter.printDate(cal);
}
 
Example 27
@Override
public void setPartitionValue(String value, int size) {
    columnVector = ColumnVector.allocate(size, dataType, MemoryMode.ON_HEAP);
    columnVector.appendInts(size, DateWritable.dateToDays(java.sql.Date.valueOf(value)));
}