org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
UnionCopier(HiveColumnVectorData columnVectorData,
            int ordinalId,
            UnionColumnVector inputVector,
            UnionVector outputVector,
            HiveOperatorContextOptions operatorContextOptions) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
  // The loop below assumes that the getChildrenFromFields() API returns
  // the list of children in the same order as was provided when building the UnionVector.
  List<FieldVector> childArrowFields = outputVector.getChildrenFromFields();
  int childPos = ordinalId + 1; // first field is immediately next to union vector itself
  for (int idx=0; idx<childArrowFields.size(); ++idx) {
    if (idx < inputVector.fields.length) {
      ColumnVector hiveFieldVector = inputVector.fields[idx];
      ValueVector arrowfieldVector = childArrowFields.get(idx);
      arrowFieldVectors.add(arrowfieldVector);
      ORCCopier childCopier = createCopier(columnVectorData, childPos, arrowfieldVector, hiveFieldVector, operatorContextOptions);
      fieldCopiers.add(childCopier);
      childPos += columnVectorData.getTotalVectorCount(childPos);
    } else {
      fieldCopiers.add(new NoOpCopier(null, null));
    }
  }
}
 
Example #2
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
UnionCopier(HiveColumnVectorData columnVectorData,
            int ordinalId,
            UnionColumnVector inputVector,
            UnionVector outputVector,
            HiveOperatorContextOptions operatorContextOptions) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
  // The loop below assumes that the getChildrenFromFields() API returns
  // the list of children in the same order as was provided when building the UnionVector.
  List<FieldVector> childArrowFields = outputVector.getChildrenFromFields();
  int childPos = ordinalId + 1; // first field is immediately next to union vector itself
  for (int idx=0; idx<childArrowFields.size(); ++idx) {
    if (idx < inputVector.fields.length) {
      ColumnVector hiveFieldVector = inputVector.fields[idx];
      ValueVector arrowfieldVector = childArrowFields.get(idx);
      arrowFieldVectors.add(arrowfieldVector);
      ORCCopier childCopier = createCopier(columnVectorData, childPos, arrowfieldVector, hiveFieldVector, operatorContextOptions);
      fieldCopiers.add(childCopier);
      childPos += columnVectorData.getTotalVectorCount(childPos);
    } else {
      fieldCopiers.add(new NoOpCopier(null, null));
    }
  }
}
 
Example #3
Source File: VectorColumnFiller.java    From secor with Apache License 2.0 6 votes vote down vote up
public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else if (value.isJsonPrimitive()) {
        UnionColumnVector vector = (UnionColumnVector) vect;
        JsonPrimitive primitive = value.getAsJsonPrimitive();

        JsonType jsonType = getJsonType(primitive);
        ConverterInfo converterInfo = childConverters.get(jsonType);
        if (converterInfo == null) {
            String message = String.format("Unable to infer type for '%s'", primitive);
            throw new IllegalArgumentException(message);
        }

        int vectorIndex = converterInfo.getVectorIndex();
        JsonConverter converter = converterInfo.getConverter();
        vector.tags[row] = vectorIndex;
        converter.convert(value, vector.fields[vectorIndex], row);
    } else {
        // It would be great to support non-primitive types in union type.
        // Let's leave this for another PR in the future.
        throw new UnsupportedOperationException();
    }
}
 
Example #4
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private ColumnVector getUnionColumnVector(UnionObjectInspector uoi) {
  ArrayList<ColumnVector> vectors = new ArrayList<>();
  List<? extends ObjectInspector> members = uoi.getObjectInspectors();
  for (ObjectInspector unionField: members) {
    vectors.add(getColumnVector(unionField));
  }
  ColumnVector[] columnVectors = vectors.toArray(new ColumnVector[0]);
  return new UnionColumnVector(VectorizedRowBatch.DEFAULT_SIZE, columnVectors);
}
 
Example #5
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private ColumnVector getUnionColumnVector(UnionObjectInspector uoi) {
  ArrayList<ColumnVector> vectors = new ArrayList<>();
  List<? extends ObjectInspector> members = uoi.getObjectInspectors();
  for (ObjectInspector unionField: members) {
    vectors.add(getColumnVector(unionField));
  }
  ColumnVector[] columnVectors = vectors.toArray(new ColumnVector[0]);
  return new UnionColumnVector(VectorizedRowBatch.DEFAULT_SIZE, columnVectors);
}
 
Example #6
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a single row of union type as a JSON object.
 *
 * @throws JSONException
 */
private static void setUnion(JSONWriter writer, UnionColumnVector vector,
                             TypeDescription schema, int row) throws JSONException {
    int tag = vector.tags[row];
    List<TypeDescription> schemaChildren = schema.getChildren();
    ColumnVector columnVector = vector.fields[tag];
    setValue(writer, columnVector, schemaChildren.get(tag), row);
}
 
Example #7
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 4 votes vote down vote up
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            setUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // To prevent similar mistakes like the one described in https://github.com/pinterest/secor/pull/1018,
            // it would be better to explicitly throw an exception here rather than ignore the incoming values,
            // which causes silent failures in a later stage.
            throw new UnsupportedOperationException();
        case MAP:
            setMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}