org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OrcBatchReader.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static void readNonNullDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			fillColumnWithRepeatingValue(vals, fieldIdx, readBigDecimal(vector.vector[0]), childCount);
		} else {
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					vals[i] = readBigDecimal(vector.vector[i]);
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
				}
			}
		}
	}
 
Example #2
Source File: OrcBatchReader.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void readNonNullDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			fillColumnWithRepeatingValue(vals, fieldIdx, readBigDecimal(vector.vector[0]), childCount);
		} else {
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					vals[i] = readBigDecimal(vector.vector[i]);
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
				}
			}
		}
	}
 
Example #3
Source File: OrcBatchReader.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void readNonNullDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			fillColumnWithRepeatingValue(vals, fieldIdx, readBigDecimal(vector.vector[0]), childCount);
		} else {
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					vals[i] = readBigDecimal(vector.vector[i]);
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
				}
			}
		}
	}
 
Example #4
Source File: OrcWriter.java    From osm2orc with ISC License 6 votes vote down vote up
@Override
public void process(WayContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
    ListColumnVector nds = (ListColumnVector) batch.cols[5];

    checkLimit();
    addCommonProperties(container);

    lat.isNull[row] = true;
    lon.isNull[row] = true;
    lat.set(row, (HiveDecimal) null);
    lon.set(row, (HiveDecimal) null);

    Way way = container.getEntity();

    nds.lengths[row] = way.getWayNodes().size();
    nds.childCount += nds.lengths[row];
    nds.child.ensureSize(nds.childCount, nds.offsets[row] != 0);

    for (int j = 0; j < way.getWayNodes().size(); j++) {
        StructColumnVector ndsStruct = (StructColumnVector) nds.child;

        ((LongColumnVector) ndsStruct.fields[0]).vector[(int) nds.offsets[row] + j] = way.getWayNodes().get(j).getNodeId();
    }
}
 
Example #5
Source File: OrcBatchReader.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static void readDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			if (vector.isNull[0]) {
				// fill vals with null values
				fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
			} else {
				// read repeating non-null value by forwarding call
				readNonNullDecimalColumn(vals, fieldIdx, vector, childCount);
			}
		} else {
			boolean[] isNullVector = vector.isNull;
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = readBigDecimal(vector.vector[i]);
					}
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
					}
				}
			}
		}
	}
 
Example #6
Source File: VectorColumnFiller.java    From secor with Apache License 2.0 5 votes vote down vote up
public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else {
        DecimalColumnVector vector = (DecimalColumnVector) vect;
        vector.vector[row].set(HiveDecimal.create(value.getAsString()));
    }
}
 
Example #7
Source File: AbstractOrcColumnVector.java    From flink with Apache License 2.0 5 votes vote down vote up
private static DecimalColumnVector createDecimalVector(int batchSize, int precision, int scale, Object value) {
	DecimalColumnVector dv = new DecimalColumnVector(batchSize, precision, scale);
	if (value == null) {
		dv.noNulls = false;
		dv.isNull[0] = true;
		dv.isRepeating = true;
	} else {
		dv.set(0, value instanceof HiveDecimal ?
				(HiveDecimal) value :
				HiveDecimal.create((BigDecimal) value));
		dv.isRepeating = true;
		dv.isNull[0] = false;
	}
	return dv;
}
 
Example #8
Source File: OrcBatchReader.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void readDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			if (vector.isNull[0]) {
				// fill vals with null values
				fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
			} else {
				// read repeating non-null value by forwarding call
				readNonNullDecimalColumn(vals, fieldIdx, vector, childCount);
			}
		} else {
			boolean[] isNullVector = vector.isNull;
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = readBigDecimal(vector.vector[i]);
					}
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
					}
				}
			}
		}
	}
 
Example #9
Source File: OrcWriter.java    From osm2orc with ISC License 5 votes vote down vote up
@Override
public void process(RelationContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
    ListColumnVector members = (ListColumnVector) batch.cols[6];

    checkLimit();
    addCommonProperties(container);

    lat.isNull[row] = true;
    lon.isNull[row] = true;
    lat.set(row, (HiveDecimal) null);
    lon.set(row, (HiveDecimal) null);

    Relation relation = container.getEntity();

    members.lengths[row] = relation.getMembers().size();
    members.childCount += members.lengths[row];
    members.child.ensureSize(members.childCount, members.offsets[row] != 0);

    for (int j = 0; j < relation.getMembers().size(); j++) {
        StructColumnVector membersStruct = (StructColumnVector) members.child;

        ((BytesColumnVector) membersStruct.fields[0]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberType().toString().toLowerCase().getBytes());
        ((LongColumnVector) membersStruct.fields[1]).vector[(int) members.offsets[row] + j] = relation.getMembers().get(j).getMemberId();
        ((BytesColumnVector) membersStruct.fields[2]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberRole().getBytes());
    }
}
 
Example #10
Source File: OrcWriter.java    From osm2orc with ISC License 5 votes vote down vote up
@Override
public void process(NodeContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];

    checkLimit();
    addCommonProperties(container);

    Node node = container.getEntity();
    lat.set(row, HiveDecimal.create(BigDecimal.valueOf(node.getLatitude())));
    lon.set(row, HiveDecimal.create(BigDecimal.valueOf(node.getLongitude())));
}
 
Example #11
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 
Example #12
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 
Example #13
Source File: OrcBatchReader.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void readDecimalColumn(Object[] vals, int fieldIdx, DecimalColumnVector vector, int childCount) {

		if (vector.isRepeating) { // fill complete column with first value
			if (vector.isNull[0]) {
				// fill vals with null values
				fillColumnWithRepeatingValue(vals, fieldIdx, null, childCount);
			} else {
				// read repeating non-null value by forwarding call
				readNonNullDecimalColumn(vals, fieldIdx, vector, childCount);
			}
		} else {
			boolean[] isNullVector = vector.isNull;
			if (fieldIdx == -1) { // set as an object
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						vals[i] = null;
					} else {
						vals[i] = readBigDecimal(vector.vector[i]);
					}
				}
			} else { // set as a field of Row
				Row[] rows = (Row[]) vals;
				for (int i = 0; i < childCount; i++) {
					if (isNullVector[i]) {
						rows[i].setField(fieldIdx, null);
					} else {
						rows[i].setField(fieldIdx, readBigDecimal(vector.vector[i]));
					}
				}
			}
		}
	}
 
Example #14
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
DecimalToVarWidthCopier(DecimalColumnVector inputVector, BaseVariableWidthVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 
Example #15
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
DecimalCopier(DecimalColumnVector inputVector, DecimalVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 
Example #16
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
DecimalToVarWidthCopier(DecimalColumnVector inputVector, BaseVariableWidthVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 
Example #17
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
DecimalCopier(DecimalColumnVector inputVector, DecimalVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 
Example #18
Source File: OrcDecimalColumnVector.java    From flink with Apache License 2.0 4 votes vote down vote up
public OrcDecimalColumnVector(DecimalColumnVector vector) {
	super(vector);
	this.vector = vector;
}
 
Example #19
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 4 votes vote down vote up
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            setUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // To prevent similar mistakes like the one described in https://github.com/pinterest/secor/pull/1018,
            // it would be better to explicitly throw an exception here rather than ignore the incoming values,
            // which causes silent failures in a later stage.
            throw new UnsupportedOperationException();
        case MAP:
            setMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}
 
Example #20
Source File: OrcConverter.java    From pentaho-hadoop-shims with Apache License 2.0 4 votes vote down vote up
protected static Object convertFromSourceToTargetDataType( ColumnVector columnVector, int currentBatchRow,
                                                           int orcValueMetaInterface ) {

  if ( columnVector.isNull[ currentBatchRow ] ) {
    return null;
  }
  switch ( orcValueMetaInterface ) {
    case ValueMetaInterface.TYPE_INET:
      try {
        return InetAddress.getByName( new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] ) );
      } catch ( UnknownHostException e ) {
        e.printStackTrace();
      }

    case ValueMetaInterface.TYPE_STRING:
      return new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );

    case ValueMetaInterface.TYPE_INTEGER:
      return (long) ( (LongColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_NUMBER:
      return ( (DoubleColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_BIGNUMBER:
      HiveDecimalWritable obj = ( (DecimalColumnVector) columnVector ).vector[ currentBatchRow ];
      return obj.getHiveDecimal().bigDecimalValue();

    case ValueMetaInterface.TYPE_TIMESTAMP:
      Timestamp timestamp = new Timestamp( ( (TimestampColumnVector) columnVector ).time[ currentBatchRow ] );
      timestamp.setNanos( ( (TimestampColumnVector) columnVector ).nanos[ currentBatchRow ] );
      return timestamp;

    case ValueMetaInterface.TYPE_DATE:
      LocalDate localDate =
        LocalDate.ofEpochDay( 0 ).plusDays( ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] );
      Date dateValue = Date.from( localDate.atStartOfDay( ZoneId.systemDefault() ).toInstant() );
      return dateValue;

    case ValueMetaInterface.TYPE_BOOLEAN:
      return ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] == 0 ? false : true;

    case ValueMetaInterface.TYPE_BINARY:
      byte[] origBytes = ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ];
      int startPos = ( (BytesColumnVector) columnVector ).start[ currentBatchRow ];
      byte[] newBytes = Arrays.copyOfRange( origBytes, startPos,
        startPos + ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );
      return newBytes;
  }

  //if none of the cases match return a null
  return null;
}