org.apache.hadoop.hive.ql.exec.vector.MapColumnVector Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.exec.vector.MapColumnVector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: VectorColumnFiller.java    From secor with Apache License 2.0 6 votes vote down vote up
public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else {
        MapColumnVector vector = (MapColumnVector) vect;
        JsonObject obj = value.getAsJsonObject();
        vector.lengths[row] = obj.size();
        vector.offsets[row] = row > 0 ? vector.offsets[row - 1] + vector.lengths[row - 1] : 0;

        // Ensure enough space is available to store the keys and the values
        vector.keys.ensureSize((int) vector.offsets[row] + obj.size(), true);
        vector.values.ensureSize((int) vector.offsets[row] + obj.size(), true);

        int i = 0;
        for (String key : obj.keySet()) {
            childConverters[0].convert(new JsonPrimitive(key), vector.keys, (int) vector.offsets[row] + i);
            childConverters[1].convert(obj.get(key), vector.values, (int) vector.offsets[row] + i);
            i++;
        }
    }
}
 
Example #2
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 5 votes vote down vote up
private static void setMap(JSONWriter writer, MapColumnVector vector,
                           TypeDescription schema, int row) throws JSONException {
    writer.object();
    List<TypeDescription> schemaChildren = schema.getChildren();
    BytesColumnVector keyVector = (BytesColumnVector) vector.keys;
    long length = vector.lengths[row];
    long offset = vector.offsets[row];
    for (int i = 0; i < length; i++) {
        writer.key(keyVector.toString((int) offset + i));
        setValue(writer, vector.values, schemaChildren.get(1), (int) offset + i);
    }
    writer.endObject();
}
 
Example #3
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
private ColumnVector getMapColumnVector(MapObjectInspector moi) {
  ColumnVector keys = getColumnVector(moi.getMapKeyObjectInspector());
  ColumnVector values = getColumnVector(moi.getMapValueObjectInspector());
  return new MapColumnVector(VectorizedRowBatch.DEFAULT_SIZE, keys, values);
}
 
Example #4
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
private ColumnVector getMapColumnVector(MapObjectInspector moi) {
  ColumnVector keys = getColumnVector(moi.getMapKeyObjectInspector());
  ColumnVector values = getColumnVector(moi.getMapValueObjectInspector());
  return new MapColumnVector(VectorizedRowBatch.DEFAULT_SIZE, keys, values);
}
 
Example #5
Source File: OrcWriter.java    From osm2orc with ISC License 4 votes vote down vote up
private void addCommonProperties(EntityContainer container) {
    LongColumnVector id = (LongColumnVector) batch.cols[0];
    BytesColumnVector type = (BytesColumnVector) batch.cols[1];
    MapColumnVector tags = (MapColumnVector) batch.cols[2];
    ListColumnVector nds = (ListColumnVector) batch.cols[5];
    ListColumnVector members = (ListColumnVector) batch.cols[6];
    LongColumnVector changeset = (LongColumnVector) batch.cols[7];
    TimestampColumnVector timestamp = (TimestampColumnVector) batch.cols[8];
    LongColumnVector uid = (LongColumnVector) batch.cols[9];
    BytesColumnVector user = (BytesColumnVector) batch.cols[10];
    LongColumnVector version = (LongColumnVector) batch.cols[11];
    LongColumnVector visible = (LongColumnVector) batch.cols[12];

    Entity entity = container.getEntity();

    id.vector[row] = entity.getId();
    changeset.vector[row] = entity.getChangesetId();
    type.setVal(row, entity.getType().toString().toLowerCase().getBytes());

    tags.offsets[row] = tags.childCount;
    tags.lengths[row] = entity.getTags().size(); // number of key/value pairings
    tags.childCount += tags.lengths[row];
    tags.keys.ensureSize(tags.childCount, tags.offsets[row] != 0);
    tags.values.ensureSize(tags.childCount, tags.offsets[row] != 0);

    int i = 0;
    for (Tag tag : entity.getTags()) {
        ((BytesColumnVector) tags.keys).setVal((int) tags.offsets[row] + i, tag.getKey().getBytes());
        ((BytesColumnVector) tags.values).setVal((int) tags.offsets[row] + i, tag.getValue().getBytes());

        i++;
    }

    timestamp.time[row] = entity.getTimestamp().getTime();
    timestamp.nanos[row] = 0;

    uid.vector[row] = entity.getUser().getId();

    user.setVal(row, entity.getUser().getName().getBytes());

    version.vector[row] = entity.getVersion();

    visible.vector[row] = 1;
    if (entity.getMetaTags().get("visible") == Boolean.FALSE) {
        visible.vector[row] = 0;
    }

    nds.offsets[row] = nds.childCount;
    nds.lengths[row] = 0;

    members.offsets[row] = members.childCount;
    members.lengths[row] = 0;
}
 
Example #6
Source File: JsonFieldFiller.java    From secor with Apache License 2.0 4 votes vote down vote up
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            setUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // To prevent similar mistakes like the one described in https://github.com/pinterest/secor/pull/1018,
            // it would be better to explicitly throw an exception here rather than ignore the incoming values,
            // which causes silent failures in a later stage.
            throw new UnsupportedOperationException();
        case MAP:
            setMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}