org.apache.hadoop.hive.serde2.io.ShortWritable Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.io.ShortWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDeepParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testHashMap() {
  final Map<Writable, Writable> map = new HashMap<Writable, Writable>();
  map.put(new IntWritable(0), new IntWritable(1));
  map.put(new IntWritable(2), new IntWritable(3));
  map.put(new IntWritable(4), new IntWritable(5));
  map.put(new IntWritable(6), new IntWritable(7));


  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
  assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new IntWritable(4)));
  assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new IntWritable(6)));
  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new ShortWritable((short) 2)));
  assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new ShortWritable((short) 4)));
  assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new ShortWritable((short) 6)));
}
 
Example #2
Source File: DataWritableWriter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private void writePrimitive(final Writable value) {
  if (value == null) {
    return;
  }
  if (value instanceof DoubleWritable) {
    recordConsumer.addDouble(((DoubleWritable) value).get());
  } else if (value instanceof BooleanWritable) {
    recordConsumer.addBoolean(((BooleanWritable) value).get());
  } else if (value instanceof FloatWritable) {
    recordConsumer.addFloat(((FloatWritable) value).get());
  } else if (value instanceof IntWritable) {
    recordConsumer.addInteger(((IntWritable) value).get());
  } else if (value instanceof LongWritable) {
    recordConsumer.addLong(((LongWritable) value).get());
  } else if (value instanceof ShortWritable) {
    recordConsumer.addInteger(((ShortWritable) value).get());
  } else if (value instanceof ByteWritable) {
    recordConsumer.addInteger(((ByteWritable) value).get());
  } else if (value instanceof BigDecimalWritable) {
    throw new UnsupportedOperationException("BigDecimal writing not implemented");
  } else if (value instanceof BinaryWritable) {
    recordConsumer.addBinary(((BinaryWritable) value).getBinary());
  } else {
    throw new IllegalArgumentException("Unknown value type: " + value + " " + value.getClass());
  }
}
 
Example #3
Source File: TestStandardParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testHashMap() {
  final Map<Writable, Writable> map = new HashMap<Writable, Writable>();
  map.put(new IntWritable(0), new IntWritable(1));
  map.put(new IntWritable(2), new IntWritable(3));
  map.put(new IntWritable(4), new IntWritable(5));
  map.put(new IntWritable(6), new IntWritable(7));

  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
  assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new IntWritable(4)));
  assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new IntWritable(6)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 2)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 4)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 6)));
}
 
Example #4
Source File: LazyShortDictionaryTreeReader.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
@Override
public Object next(Object previous) throws IOException {
  ShortWritable result = null;
  if (valuePresent) {
    result = createWritable(previous, readShort());
  }
  return result;
}
 
Example #5
Source File: LazyShortDirectTreeReader.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
@Override
public Object next(Object previous) throws IOException {
  ShortWritable result = null;
  if (valuePresent) {
    result = createWritable(previous, readShort());
  }
  return result;
}
 
Example #6
Source File: TestDeepParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegularMap() {
  final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
  final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};

  final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
    new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});

  final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});

  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new ShortWritable((short) 2)));
}
 
Example #7
Source File: ParquetShortInspector.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Override
public short get(Object o) {
  // Accept int writables and convert them.
  if (o instanceof IntWritable) {
    return (short) ((IntWritable) o).get();
  }
  return ((ShortWritable) o).get();
}
 
Example #8
Source File: ParquetHiveSerDe.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
    throws SerDeException {
  if (obj == null) {
    return null;
  }
  switch (inspector.getPrimitiveCategory()) {
  case VOID:
    return null;
  case BOOLEAN:
    return new BooleanWritable(((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
  case BYTE:
    return new ByteWritable((byte) ((ByteObjectInspector) inspector).get(obj));
  case DOUBLE:
    return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
  case FLOAT:
    return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
  case INT:
    return new IntWritable(((IntObjectInspector) inspector).get(obj));
  case LONG:
    return new LongWritable(((LongObjectInspector) inspector).get(obj));
  case SHORT:
    return new ShortWritable((short) ((ShortObjectInspector) inspector).get(obj));
  case STRING:
    return new BinaryWritable(Binary.fromString(((StringObjectInspector) inspector).getPrimitiveJavaObject(obj)));
  default:
    throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
  }
}
 
Example #9
Source File: HiveWritableValueWriter.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public Result write(Writable writable, Generator generator) {
    if (writable instanceof ByteWritable) {
        generator.writeNumber(((ByteWritable) writable).get());
    }
    else if (writable instanceof DoubleWritable) {
        generator.writeNumber(((DoubleWritable) writable).get());
    }
    else if (writable instanceof ShortWritable) {
        generator.writeNumber(((ShortWritable) writable).get());
    }
    // HiveDecimal - Hive 0.11+
    else if (writable != null && HiveConstants.DECIMAL_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(writable.toString());
    }
    // pass the UNIX epoch
    else if (writable instanceof TimestampWritable) {
        long ts = ((TimestampWritable) writable).getTimestamp().getTime();
        Calendar cal = Calendar.getInstance();
        cal.setTimeInMillis(ts);
        generator.writeString(DatatypeConverter.printDateTime(cal));
    }
    // HiveDate - Hive 0.12+
    else if (writable != null && HiveConstants.DATE_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(DateWritableWriter.toES(writable));
    }
    // HiveVarcharWritable - Hive 0.12+
    else if (writable != null && HiveConstants.VARCHAR_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(writable.toString());
    }
    // HiveChar - Hive 0.13+
    else if (writable != null && HiveConstants.CHAR_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(StringUtils.trim(writable.toString()));
    }
    else {
        return super.write(writable, generator);
    }

    return Result.SUCCESFUL();
}
 
Example #10
Source File: TestStandardParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegularMap() {
  final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)};
  final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)};

  final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{
    new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)});

  final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap});

  assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0)));
  assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0)));
  assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 2)));
}
 
Example #11
Source File: BaseJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Copies the Writable at fieldIndex from rowBase to row, then sets the value of the Writable
 * to the value in parser
 * 
 * @param fieldIndex column index of field in row
 * @param parser JsonParser pointing to the attribute
 * @throws JsonParseException
 * @throws IOException
 */
private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{

	PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);
	if (JsonToken.VALUE_NULL == parser.getCurrentToken())
		return;  // leave the row-cell as null

	// set the field in the row to the writable from rowBase
	row.set(fieldIndex, rowBase.get(fieldIndex));

	switch (poi.getPrimitiveCategory()){
	case BYTE:
		((ByteWritable)row.get(fieldIndex)).set(parser.getByteValue());
		break;
	case SHORT:
		((ShortWritable)row.get(fieldIndex)).set(parser.getShortValue());
		break;
	case INT:
		((IntWritable)row.get(fieldIndex)).set(parser.getIntValue());
		break;
	case LONG:
		((LongWritable)row.get(fieldIndex)).set(parser.getLongValue());
		break;
	case DOUBLE:
		((DoubleWritable)row.get(fieldIndex)).set(parser.getDoubleValue());
		break;
	case FLOAT:
		((FloatWritable)row.get(fieldIndex)).set(parser.getFloatValue());
		break;
	case BOOLEAN:
		((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
		break;
	case DATE:    // DateWritable stores days not milliseconds.
		((DateWritable)row.get(fieldIndex)).set(parseDate(parser));
		break;
	case TIMESTAMP:
		((TimestampWritable)row.get(fieldIndex)).set(parseTime(parser));
		break;
	default:    // STRING/unrecognized
		((Text)row.get(fieldIndex)).set(parser.getText());
		break;	
	}
}
 
Example #12
Source File: TestOrcFile.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeepCopy() throws Exception {
  // Create a table and write a row to it
  ObjectInspector inspector;
  synchronized (TestOrcFile.class) {
    inspector = ObjectInspectorFactory.getReflectionObjectInspector
        (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
  }
  ReaderWriterProfiler.setProfilerOptions(conf);
  Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
      100000, CompressionKind.ZLIB, 10000, 10000);
  writer.addRow(new BigRow(false, (byte) 1, (short) 1, 1,
      1L, (float) 1.0, 1.0, bytes(1), "1",
      new MiddleStruct(inner(1, "bye"), inner(2, "sigh")),
      list(inner(3, "good"), inner(4, "bad")),
      map(inner(3, "good"), inner(4, "bad"))));

  writer.close();

  // Prepare to tread back the row
  ReaderWriterProfiler.setProfilerOptions(conf);
  Reader reader = OrcFile.createReader(fs, testFilePath, conf);
  RecordReader rows = reader.rows(null);
  OrcLazyStruct lazyRow = null;
  OrcStruct row = null;
  lazyRow = (OrcLazyStruct) rows.next(lazyRow);
  row = (OrcStruct) lazyRow.materialize();

  // Check that the object read equals what is expected, then copy the object, and make the same
  // check
  OrcLazyObject obj;
  assertEquals(false,
      ((BooleanWritable) ((OrcLazyBoolean) row.getFieldValue(0)).materialize()).get());
  obj = new OrcLazyBoolean((OrcLazyBoolean) row.getFieldValue(0));
  assertEquals(false, ((BooleanWritable) obj.materialize()).get());

  assertEquals(1, ((ByteWritable) ((OrcLazyByte) row.getFieldValue(1)).materialize()).get());
  obj = new OrcLazyByte((OrcLazyByte) row.getFieldValue(1));
  assertEquals(1, ((ByteWritable) obj.materialize()).get());

  assertEquals(1, ((ShortWritable) ((OrcLazyShort) row.getFieldValue(2)).materialize()).get());
  obj = new OrcLazyShort((OrcLazyShort) row.getFieldValue(2));
  assertEquals(1, ((ShortWritable) obj.materialize()).get());

  assertEquals(1, ((IntWritable) ((OrcLazyInt) row.getFieldValue(3)).materialize()).get());
  obj = new OrcLazyInt((OrcLazyInt) row.getFieldValue(3));
  assertEquals(1, ((IntWritable) obj.materialize()).get());

  assertEquals(1, ((LongWritable) ((OrcLazyLong) row.getFieldValue(4)).materialize()).get());
  obj = new OrcLazyLong((OrcLazyLong) row.getFieldValue(4));
  assertEquals(1, ((LongWritable) obj.materialize()).get());

  assertEquals(1.0f,
      ((FloatWritable) ((OrcLazyFloat) row.getFieldValue(5)).materialize()).get());
  obj = new OrcLazyFloat((OrcLazyFloat) row.getFieldValue(5));
  assertEquals(1.0f, ((FloatWritable) obj.materialize()).get());

  assertEquals(1.0,
      ((DoubleWritable) ((OrcLazyDouble) row.getFieldValue(6)).materialize()).get());
  obj = new OrcLazyDouble((OrcLazyDouble) row.getFieldValue(6));
  assertEquals(1.0, ((DoubleWritable) obj.materialize()).get());

  assertEquals(bytes(1), ((OrcLazyBinary) row.getFieldValue(7)).materialize());
  obj = new OrcLazyBinary((OrcLazyBinary) row.getFieldValue(7));
  assertEquals(bytes(1), obj.materialize());

  assertEquals("1", ((Text) ((OrcLazyString) row.getFieldValue(8)).materialize()).toString());
  obj = new OrcLazyString((OrcLazyString) row.getFieldValue(8));
  assertEquals("1", ((Text) obj.materialize()).toString());

  // Currently copies are not supported for complex types
}
 
Example #13
Source File: JsonSerDeTestingBase.java    From spatial-framework-for-hadoop with Apache License 2.0 4 votes vote down vote up
protected void addWritable(ArrayList<Object> stuff, short item) {
	stuff.add(new ShortWritable(item));
}
 
Example #14
Source File: TestEsriJsonSerDe.java    From spatial-framework-for-hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void TestColumnTypes() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "flag,num1,num2,text");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,string");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

	// {"attributes":{"flag":false,"num":"5","text":"Point(15.0 5.0)"}}
       addWritable(stuff, false);
       addWritable(stuff, (byte)2);
       addWritable(stuff, (short)5);
       addWritable(stuff, "Point(15.0 5.0)");
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("flag", row, rowOI);
	Assert.assertEquals(false, ((BooleanWritable)fieldData).get());
	fieldData = getField("num1", row, rowOI);
	Assert.assertEquals((byte)2, ((ByteWritable)fieldData).get());
	fieldData = getField("num2", row, rowOI);
	Assert.assertEquals((short)5, ((ShortWritable)fieldData).get());
	fieldData = getField("text", row, rowOI);
	Assert.assertEquals("Point(15.0 5.0)", ((Text)fieldData).toString());

	stuff.set(0, new BooleanWritable(true));
	stuff.set(1, new ByteWritable((byte)4));
	stuff.set(2, new ShortWritable((short)4));
	//stuff.set(3, new Text("other"));
	LazyStringObjectInspector loi = LazyPrimitiveObjectInspectorFactory.
		getLazyStringObjectInspector(false, (byte)'\0');
	LazyString lstr = new LazyString(loi);
	ByteArrayRef bar = new ByteArrayRef();
	bar.setData("other".getBytes());
	lstr.init(bar, 0, 5);
	stuff.set(3, lstr);
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("flag", row, rowOI);
	Assert.assertEquals(true, ((BooleanWritable)fieldData).get());
	fieldData = getField("num1", row, rowOI);
	Assert.assertEquals((byte)4, ((ByteWritable)fieldData).get());
	fieldData = getField("num2", row, rowOI);
	Assert.assertEquals((short)4, ((ShortWritable)fieldData).get());
	fieldData = getField("text", row, rowOI);
	Assert.assertEquals("other", ((Text)fieldData).toString());
}
 
Example #15
Source File: HiveValueReader.java    From elasticsearch-hadoop with Apache License 2.0 4 votes vote down vote up
@Override
protected Class<? extends Writable> shortType() {
    return ShortWritable.class;
}
 
Example #16
Source File: HiveValueReader.java    From elasticsearch-hadoop with Apache License 2.0 4 votes vote down vote up
@Override
protected Object processShort(Short value) {
    return new ShortWritable(value);
}
 
Example #17
Source File: HiveTypeToJsonTest.java    From elasticsearch-hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testShort() {
    // byte is not recognized by the schema
    assertEquals("32767", hiveTypeToJson(new MyHiveType(new ShortWritable(Short.MAX_VALUE), shortTypeInfo)));
}
 
Example #18
Source File: OrcLazyShortObjectInspector.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
@Override
public short get(Object o) {
  return ((ShortWritable)getPrimitiveWritableObject(o)).get();
}
 
Example #19
Source File: OrcLazyShortObjectInspector.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
@Override
public Object getPrimitiveJavaObject(Object o) {
  ShortWritable writable = getPrimitiveWritableObject(o);
  return writable == null ? null : Short.valueOf(writable.get());
}
 
Example #20
Source File: OrcTester.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    }
    else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}
 
Example #21
Source File: WebServiceInterface.java    From quetzal with Eclipse Public License 2.0 4 votes vote down vote up
default public List<Object[]> parseResponse(InputStream is, NamespaceResolver namespace, String xPathForRows, List<List<String>> xPathForEachColumn) throws Exception {		
		
		System.out.println("In parse response");
		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		factory.setNamespaceAware(true);

		DocumentBuilder builder = factory.newDocumentBuilder(); 

		Document doc = builder.parse(new InputSource(is));
	//	printDocument(doc);
		XPath xPath = XPathFactory.newInstance().newXPath();

	    xPath.setNamespaceContext(namespace);
		
		NodeList rows = (NodeList) xPath.compile(xPathForRows).evaluate(doc,
				XPathConstants.NODESET);

		List<Object[]> result = new LinkedList<Object[]>();
		for (int i = 0; i < rows.getLength(); i++) {
			Node row = rows.item(i);
			Object[] k = null;
			k = new Object[xPathForEachColumn.size() * 2];
			
			IntWritable indexValue = null;
			
			for (int j = 0; j < xPathForEachColumn.size(); j++) {
				List<String> p = xPathForEachColumn.get(j);
				String xPathForColValue = p.get(1);
				String xPathForColType = p.get(2);
				if (xPathForColValue.isEmpty()) {
					xPathForColValue = "./s:binding[./@name='" + p.get(0) + "']";
					xPathForColType = "./s:binding[./@name='" + p.get(0) + "']//@datatype";
				}
				Node column = (Node) xPath.compile(xPathForColValue).evaluate(row,
						XPathConstants.NODE);
		//		System.out.println("GOT HERE:" + xPathForColValue + " " + column);
				String value = column.getTextContent();
				String type = null;
				if (xPathForColType.equals(String.valueOf(TypeMap.IRI_ID))) {
					type = TypeMap.IRI_TYPE_IRI;
				} else if (xPathForColType.startsWith(".") || xPathForColType.startsWith("/")) {
					Node t = ((Node) xPath.compile(xPathForColType).evaluate(row,
							XPathConstants.NODE));
					if (t != null) {
						type = ((Node) xPath.compile(xPathForColType).evaluate(row,
							XPathConstants.NODE)).getTextContent();
					} else {
						type = xPathForColType;
					}
				} else {
					type = xPathForColType;
				}
				
				k[j * 2] = new Text(value.trim());
				ShortWritable st = new ShortWritable();
				st.set(getTypedValue(value, type));
				k[(j * 2) +1] =  st;
			}

			row.getParentNode().removeChild(row);
			result.add(k);
			
		}
		System.out.println("Parsed response");
//		br.close();
		is.close();
		
//		prettyPrint(result);
		return result;
	}
 
Example #22
Source File: TestOrcStorage.java    From spork with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
private void compareData(Object expected, Object actual) {
    if (expected instanceof Text) {
        assertEquals(String.class, actual.getClass());
        assertEquals(expected.toString(), actual);
    } else if (expected instanceof ShortWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int)((ShortWritable) expected).get(), actual);
    } else if (expected instanceof IntWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals(((IntWritable) expected).get(), actual);
    } else if (expected instanceof LongWritable) {
        assertEquals(Long.class, actual.getClass());
        assertEquals(((LongWritable) expected).get(), actual);
    } else if (expected instanceof FloatWritable) {
        assertEquals(Float.class, actual.getClass());
        assertEquals(((FloatWritable) expected).get(), actual);
    } else if (expected instanceof HiveDecimalWritable) {
        assertEquals(BigDecimal.class, actual.getClass());
        assertEquals(((HiveDecimalWritable) expected).toString(), actual.toString());
    } else if (expected instanceof DoubleWritable) {
        assertEquals(Double.class, actual.getClass());
        assertEquals(((DoubleWritable) expected).get(), actual);
    } else if (expected instanceof BooleanWritable) {
        assertEquals(Boolean.class, actual.getClass());
        assertEquals(((BooleanWritable) expected).get(), actual);
    } else if (expected instanceof TimestampWritable) {
        assertEquals(DateTime.class, actual.getClass());
        assertEquals(((TimestampWritable) expected).getTimestamp().getTime(),
                ((DateTime) actual).getMillis());
    } else if (expected instanceof BytesWritable) {
        assertEquals(DataByteArray.class, actual.getClass());
        BytesWritable bw = (BytesWritable) expected;
        assertEquals(new DataByteArray(bw.getBytes(), 0, bw.getLength()), actual);
    } else if (expected instanceof ByteWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int) ((ByteWritable) expected).get(), actual);
    } else if (expected instanceof OrcStruct) {
        assertEquals(BinSedesTuple.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof ArrayList) {
        assertEquals(DefaultDataBag.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof HashMap) {
        assertEquals(HashMap.class, actual.getClass());
        assertEquals(((HashMap) expected).size(), ((HashMap) actual).size());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected == null) {
        assertEquals(expected, actual);
    } else {
        Assert.fail("Unknown object type: " + expected.getClass().getName());
    }
}
 
Example #23
Source File: TestDeepParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Test
public void testEmptyContainer() {
  final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new ArrayWritable[0]);
  assertNull("Should be null", inspector.getMapValueElement(map, new ShortWritable((short) 0)));
}
 
Example #24
Source File: TestDeepParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Test
public void testNullContainer() {
  final ArrayWritable map = new ArrayWritable(ArrayWritable.class, null);
  assertNull("Should be null", inspector.getMapValueElement(map, new ShortWritable((short) 0)));
}
 
Example #25
Source File: TestDeepParquetHiveMapInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Test
public void testNullMap() {
  assertNull("Should be null", inspector.getMapValueElement(null, new ShortWritable((short) 0)));
}
 
Example #26
Source File: TestParquetSerDe.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public void testParquetHiveSerDe() throws Throwable {
  try {
    // Create the SerDe
    System.out.println("test: testParquetHiveSerDe");

    final ParquetHiveSerDe serDe = new ParquetHiveSerDe();
    final Configuration conf = new Configuration();
    final Properties tbl = createProperties();
    serDe.initialize(conf, tbl);

    // Data
    final Writable[] arr = new Writable[8];

    arr[0] = new ByteWritable((byte) 123);
    arr[1] = new ShortWritable((short) 456);
    arr[2] = new IntWritable(789);
    arr[3] = new LongWritable(1000l);
    arr[4] = new DoubleWritable((double) 5.3);
    arr[5] = new BinaryWritable(Binary.fromString("hive and hadoop and parquet. Big family."));

    final Writable[] mapContainer = new Writable[1];
    final Writable[] map = new Writable[3];
    for (int i = 0; i < 3; ++i) {
      final Writable[] pair = new Writable[2];
      pair[0] = new BinaryWritable(Binary.fromString("key_" + i));
      pair[1] = new IntWritable(i);
      map[i] = new ArrayWritable(Writable.class, pair);
    }
    mapContainer[0] = new ArrayWritable(Writable.class, map);
    arr[6] = new ArrayWritable(Writable.class, mapContainer);

    final Writable[] arrayContainer = new Writable[1];
    final Writable[] array = new Writable[5];
    for (int i = 0; i < 5; ++i) {
      array[i] = new BinaryWritable(Binary.fromString("elem_" + i));
    }
    arrayContainer[0] = new ArrayWritable(Writable.class, array);
    arr[7] = new ArrayWritable(Writable.class, arrayContainer);

    final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr);
    // Test
    deserializeAndSerializeLazySimple(serDe, arrWritable);
    System.out.println("test: testParquetHiveSerDe - OK");

  } catch (final Throwable e) {
    e.printStackTrace();
    throw e;
  }
}
 
Example #27
Source File: ParquetShortInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Override
public Object set(final Object o, final short val) {
  ((ShortWritable) o).set(val);
  return o;
}
 
Example #28
Source File: ParquetShortInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Override
public Object create(final short val) {
  return new ShortWritable(val);
}
 
Example #29
Source File: ParquetShortInspector.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
@Override
public Object getPrimitiveWritableObject(final Object o) {
  return o == null ? null : new ShortWritable(get(o));
}
 
Example #30
Source File: TestCacheableObjectInspectorConverters.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Test
public void testObjectInspectorConverters() throws Throwable {
  try {
    CacheableObjectInspectorConverters cacheableObjectInspectorConverters = new CacheableObjectInspectorConverters();

    // Boolean
    Converter booleanConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableBooleanObjectInspector);
    Assert.assertEquals(new BooleanWritable(false), booleanConverter.convert(Integer.valueOf(0)), "BooleanConverter");
    Assert.assertEquals(new BooleanWritable(true), booleanConverter.convert(Integer.valueOf(1)), "BooleanConverter");
    Assert.assertEquals(null, booleanConverter.convert(null), "BooleanConverter");

    // Byte
    Converter byteConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableByteObjectInspector);
    Assert.assertEquals(new ByteWritable((byte) 0), byteConverter.convert(Integer.valueOf(0)), "ByteConverter");
    Assert.assertEquals(new ByteWritable((byte) 1), byteConverter.convert(Integer.valueOf(1)), "ByteConverter");
    Assert.assertEquals(null, byteConverter.convert(null), "ByteConverter");

    // Short
    Converter shortConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableShortObjectInspector);
    Assert.assertEquals(new ShortWritable((short) 0), shortConverter.convert(Integer.valueOf(0)), "ShortConverter");
    Assert.assertEquals(new ShortWritable((short) 1), shortConverter.convert(Integer.valueOf(1)), "ShortConverter");
    Assert.assertEquals(null, shortConverter.convert(null), "ShortConverter");

    // Int
    Converter intConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableIntObjectInspector);
    Assert.assertEquals(new IntWritable(0), intConverter.convert(Integer.valueOf(0)), "IntConverter");
    Assert.assertEquals(new IntWritable(1), intConverter.convert(Integer.valueOf(1)), "IntConverter");
    Assert.assertEquals(null, intConverter.convert(null), "IntConverter");

    // Long
    Converter longConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableLongObjectInspector);
    Assert.assertEquals(new LongWritable(0), longConverter.convert(Integer.valueOf(0)), "LongConverter");
    Assert.assertEquals(new LongWritable(1), longConverter.convert(Integer.valueOf(1)), "LongConverter");
    Assert.assertEquals(null, longConverter.convert(null), "LongConverter");

    // Float
    Converter floatConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableFloatObjectInspector);
    Assert.assertEquals(new FloatWritable(0), floatConverter.convert(Integer.valueOf(0)), "FloatConverter");
    Assert.assertEquals(new FloatWritable(1), floatConverter.convert(Integer.valueOf(1)), "FloatConverter");
    Assert.assertEquals(null, floatConverter.convert(null), "FloatConverter");

    // Double
    Converter doubleConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableDoubleObjectInspector);
    Assert.assertEquals(new DoubleWritable(0), doubleConverter.convert(Integer.valueOf(0)), "DoubleConverter");
    Assert.assertEquals(new DoubleWritable(1), doubleConverter.convert(Integer.valueOf(1)), "DoubleConverter");
    Assert.assertEquals(null, doubleConverter.convert(null), "DoubleConverter");

    // Text
    Converter textConverter =
        cacheableObjectInspectorConverters.getConverter(javaIntObjectInspector, writableStringObjectInspector);
    Assert.assertEquals(new Text("0"), textConverter.convert(Integer.valueOf(0)), "TextConverter");
    Assert.assertEquals(new Text("1"), textConverter.convert(Integer.valueOf(1)), "TextConverter");
    Assert.assertEquals(null, textConverter.convert(null), "TextConverter");

    textConverter =
        cacheableObjectInspectorConverters.getConverter(writableBinaryObjectInspector, writableStringObjectInspector);
    Assert.assertEquals(new Text("hive"), textConverter
        .convert(new BytesWritable(new byte[]{(byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e'})), "TextConverter");
    Assert.assertEquals(null, textConverter.convert(null), "TextConverter");

    textConverter =
        cacheableObjectInspectorConverters.getConverter(writableStringObjectInspector, writableStringObjectInspector);
    Assert.assertEquals(new Text("hive"), textConverter.convert(new Text("hive")), "TextConverter");
    Assert.assertEquals(null, textConverter.convert(null), "TextConverter");

    textConverter =
        cacheableObjectInspectorConverters.getConverter(javaStringObjectInspector, writableStringObjectInspector);
    Assert.assertEquals(new Text("hive"), textConverter.convert(new String("hive")), "TextConverter");
    Assert.assertEquals(null, textConverter.convert(null), "TextConverter");

    textConverter = cacheableObjectInspectorConverters.getConverter(javaHiveDecimalObjectInspector,
        writableStringObjectInspector);
    Assert.assertEquals(new Text("100.001"), textConverter.convert(HiveDecimal.create("100.001")), "TextConverter");
    Assert.assertEquals(null, textConverter.convert(null), "TextConverter");

    // Binary
    Converter baConverter =
        cacheableObjectInspectorConverters.getConverter(javaStringObjectInspector, writableBinaryObjectInspector);
    Assert.assertEquals(new BytesWritable(new byte[]{(byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e'}),
        baConverter.convert("hive"), "BAConverter");
    Assert.assertEquals(null, baConverter.convert(null), "BAConverter");

    baConverter =
        cacheableObjectInspectorConverters.getConverter(writableStringObjectInspector, writableBinaryObjectInspector);
    Assert.assertEquals(new BytesWritable(new byte[]{(byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e'}),
        baConverter.convert(new Text("hive")), "BAConverter");
    Assert.assertEquals(null, baConverter.convert(null), "BAConverter");
  } catch (Throwable e) {
    e.printStackTrace();
    throw e;
  }
}