Java Code Examples for org.apache.hadoop.hive.serde2.AbstractSerDe#deserialize()
The following examples show how to use
org.apache.hadoop.hive.serde2.AbstractSerDe#deserialize() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"properties\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example 2
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestDateParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2020-02-20",
((DateWritable)fieldData).get().toString());
value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2017-05-05",
((DateWritable)fieldData).get().toString());
}
Example 3
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"properties\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example 4
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"type\":\"Point\",\"coordinates\":[7.0,4.0]}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example 5
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":null}");
value.set("{\"attributes\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"attributes\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example 6
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestDateParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2020-02-20",
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":\"2017-05-05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2017-05-05",
((DateWritable)fieldData).get().toString());
}
Example 7
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example 8
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example 9
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void TestTimeParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "timestamp");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd").parse("2020-02-20").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-05-05 05:05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm").parse("2017-05-05 05:05").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-08-09 10:11:12\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2017-08-09 10:11:12").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime());
value.set("{\"attributes\":{\"when\":\"2017-06-05 04:03:02.123456789\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(
new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse("2017-06-05 04:03:02.123").getTime(),
((TimestampWritable)fieldData).getTimestamp().getTime()); // ns parsed but not checked
}
Example 10
| Source File: HiveTextReader.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Override
public int populateData() throws IOException, SerDeException {
final SkipRecordsInspector skipRecordsInspector = this.skipRecordsInspector;
final RecordReader<Object, Object> reader = this.reader;
final Converter partTblObjectInspectorConverter = this.partTblObjectInspectorConverter;
final Object key = this.key;
final int numRowsPerBatch = (int) this.numRowsPerBatch;
final StructField[] selectedStructFieldRefs = this.selectedStructFieldRefs;
final AbstractSerDe partitionSerDe = this.partitionSerDe;
final StructObjectInspector finalOI = this.finalOI;
final ObjectInspector[] selectedColumnObjInspectors = this.selectedColumnObjInspectors;
final HiveFieldConverter[] selectedColumnFieldConverters = this.selectedColumnFieldConverters;
final ValueVector[] vectors = this.vectors;
skipRecordsInspector.reset();
Object value;
int recordCount = 0;
while (recordCount < numRowsPerBatch) {
try (OperatorStats.WaitRecorder recorder = OperatorStats.getWaitRecorder(this.context.getStats())) {
boolean hasNext = reader.next(key, value = skipRecordsInspector.getNextValue());
if (!hasNext) {
break;
}
}
catch(FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
if (skipRecordsInspector.doSkipHeader(recordCount++)) {
continue;
}
Object bufferedValue = skipRecordsInspector.bufferAdd(value);
if (bufferedValue != null) {
Object deSerializedValue = partitionSerDe.deserialize((Writable) bufferedValue);
if (partTblObjectInspectorConverter != null) {
deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue);
}
for (int i = 0; i < selectedStructFieldRefs.length; i++) {
Object hiveValue = finalOI.getStructFieldData(deSerializedValue, selectedStructFieldRefs[i]);
if (hiveValue != null) {
selectedColumnFieldConverters[i].setSafeValue(selectedColumnObjInspectors[i], hiveValue, vectors[i], skipRecordsInspector.getActualCount());
}
}
skipRecordsInspector.incrementActualCount();
}
skipRecordsInspector.incrementTempCount();
}
for (int i = 0; i < selectedStructFieldRefs.length; i++) {
vectors[i].setValueCount(skipRecordsInspector.getActualCount());
}
skipRecordsInspector.updateContinuance();
return skipRecordsInspector.getActualCount();
}
Example 11
| Source File: JsonSerDeTestingBase.java From spatial-framework-for-hadoop with Apache License 2.0 | 4 votes |
|
protected Object runSerDe(Object stuff, AbstractSerDe jserde, StructObjectInspector rowOI) throws Exception {
Writable jsw = jserde.serialize(stuff, rowOI);
//System.err.println(jsw);
return jserde.deserialize(jsw);
}
