org.apache.hadoop.io.ShortWritable Java Examples

The following examples show how to use org.apache.hadoop.io.ShortWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OrcTestTools.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * All Writable objects passed in here are guaranteed to be primitive writable objects.
 */
private boolean objCastHelper(Object javaObj, Writable obj) {
  if (obj instanceof IntWritable) {
    return ((IntWritable) obj).get() == (Integer) javaObj;
  } else if (obj instanceof Text) {
    return (obj).toString().equals(javaObj);
  } else if (obj instanceof LongWritable) {
    return ((LongWritable) obj).get() == (Long) javaObj;
  } else if (obj instanceof ShortWritable) {
    return ((ShortWritable) obj).get() == (Short) javaObj;
  } else if (obj instanceof DoubleWritable) {
    return ((DoubleWritable) obj).get() == (Double) javaObj;
  } else {
    throw new RuntimeException("Cannot recognize the writable type, please enrich the castHelper function");
  }
}
 
Example #2
Source File: FactDistinctColumnsJob.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void setupMapper(String intermediateTable) throws IOException {
//        FileInputFormat.setInputPaths(job, input);

        String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
        HCatInputFormat.setInput(job, dbTableNames[0],
                dbTableNames[1]);
        
        job.setInputFormatClass(HCatInputFormat.class);
        job.setMapperClass(FactDistinctColumnsMapper.class);
        job.setCombinerClass(FactDistinctColumnsCombiner.class);
        job.setMapOutputKeyClass(ShortWritable.class);
        job.setMapOutputValueClass(Text.class);
    }
 
Example #3
Source File: CustomWritable.java    From pxf with Apache License 2.0 4 votes vote down vote up
@Override
public void write(DataOutput out) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.write(out);

    // 1. num, int1, int2
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.set(num[i]);
        intw.write(out);
    }

    intw.set(int1);
    intw.write(out);

    intw.set(int2);
    intw.write(out);

    // 2. st1
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.set(strings[i]);
        txt.write(out);
    }

    txt.set(st1);
    txt.write(out);

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.set(dubs[i]);
        dw.write(out);
    }

    dw.set(db);
    dw.write(out);

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.set(fts[i]);
        fw.write(out);
    }

    fw.set(ft);
    fw.write(out);

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.set(lngs[i]);
        lw.write(out);
    }
    lw.set(lng);
    lw.write(out);

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.set(bools[i]);
        bw.write(out);
    }
    bw.set(bool);
    bw.write(out);

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.set(shrts[i]);
        sw.write(out);
    }
    sw.set(shrt);
    sw.write(out);

    // 8. bytes
    // BytesWritable btsw = new BytesWritable(bts);
    // btsw.write(out);
    BytesWritable btsw = new BytesWritable();
    btsw.setCapacity(bts.length);
    btsw.setSize(bts.length);
    btsw.set(bts, 0, bts.length);
    btsw.write(out);
}
 
Example #4
Source File: CustomWritable.java    From pxf with Apache License 2.0 4 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.readFields(in);
    tms = tms_text.toString();

    // 1. integers
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.readFields(in);
        num[i] = intw.get();
    }

    intw.readFields(in);
    int1 = intw.get();

    intw.readFields(in);
    int2 = intw.get();

    // 2. strings
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.readFields(in);
        strings[i] = txt.toString();
    }

    txt.readFields(in);
    st1 = txt.toString();

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.readFields(in);
        dubs[i] = dw.get();
    }

    dw.readFields(in);
    db = dw.get();

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.readFields(in);
        fts[i] = fw.get();
    }

    fw.readFields(in);
    ft = fw.get();

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.readFields(in);
        lngs[i] = lw.get();
    }

    lw.readFields(in);
    lng = lw.get();

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.readFields(in);
        bools[i] = bw.get();
    }

    bw.readFields(in);
    bool = bw.get();

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.readFields(in);
        shrts[i] = sw.get();
    }
    sw.readFields(in);
    shrt = sw.get();

    // 8. bytes
    BytesWritable btsw = new BytesWritable();
    btsw.readFields(in);
    byte[] buffer = btsw.getBytes();
    bts = new byte[btsw.getLength()];
    for (int i = 0; i < btsw.getLength(); i++) {
        bts[i] = buffer[i];
    }
}
 
Example #5
Source File: FSImageSerialization.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/** read short value */
static short readShort(DataInput in) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.readFields(in);
  return uShort.get();
}
 
Example #6
Source File: FSImageSerialization.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/** write short value */
static void writeShort(short value, DataOutputStream out) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.set(value);
  uShort.write(out);
}
 
Example #7
Source File: FSImageSerialization.java    From big-c with Apache License 2.0 4 votes vote down vote up
/** read short value */
static short readShort(DataInput in) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.readFields(in);
  return uShort.get();
}
 
Example #8
Source File: FSImageSerialization.java    From big-c with Apache License 2.0 4 votes vote down vote up
/** write short value */
static void writeShort(short value, DataOutputStream out) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.set(value);
  uShort.write(out);
}
 
Example #9
Source File: WritableUtils.java    From warp10-platform with Apache License 2.0 4 votes vote down vote up
public static Writable toWritable(Object o) throws IOException {
  if (o instanceof Long) {
    return new LongWritable(((Long) o).longValue());
  } else if (o instanceof String) {
    return new Text(o.toString());
  } else if (o instanceof byte[]) {
    return new BytesWritable((byte[]) o);
  } else if (o instanceof Integer) {
    return new IntWritable(((Integer) o).intValue());
  } else if (o instanceof Short) {
    return new ShortWritable(((Short) o).shortValue());
  } else if (o instanceof Byte) {
    return new ByteWritable(((Byte) o).byteValue());
  } else if (o instanceof Double) {
    return new DoubleWritable(((Double) o).doubleValue());
  } else if (o instanceof Float) {
    return new FloatWritable(((Float) o).floatValue());
  } else if (o instanceof Boolean) {
    return new BooleanWritable(((Boolean) o).booleanValue());
  } else if (o instanceof List) {
    Writable[] a = new Writable[((List) o).size()];
    for (int i = 0; i < a.length; i++) {
      a[i] = new ObjectWritable(toWritable(((List) o).get(i)));
    }
    return new ArrayWritable(ObjectWritable.class, a);
  } else if (o instanceof Map) {
    MapWritable map = new MapWritable();
    for (Entry<Object,Object> entry: ((Map<Object,Object>) o).entrySet()) {
      map.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
    }
    return map;
  } else if (null == o) {
    return NullWritable.get();
  } else {
    ObjectWritable ow = new ObjectWritable();
    ow.set(o);
    return ow;
  }// else {
  //  throw new IOException("Unsupported type " + o.getClass());
  //}
}
 
Example #10
Source File: OrcUtils.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
/**
 * Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
 */
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
  if (from instanceof ByteWritable) {
    if (to instanceof ByteWritable) {
      ((ByteWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ByteWritable) from).get());
      return;
    }
  } else if (from instanceof ShortWritable) {
    if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ShortWritable) from).get());
      return;
    }
  } else if (from instanceof IntWritable) {
    if (to instanceof IntWritable) {
      ((IntWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((IntWritable) from).get());
      return;
    }
  } else if (from instanceof LongWritable) {
    if (to instanceof LongWritable) {
      ((LongWritable) to).set(((LongWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((LongWritable) from).get());
      return;
    }
    // Following from this branch, type-widening is not allowed and only value-copy will happen.
  } else if (from instanceof DoubleWritable) {
    if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((DoubleWritable) from).get());
      return;
    }
  } else if (from instanceof BytesWritable) {
    if (to instanceof BytesWritable) {
      ((BytesWritable) to).set((BytesWritable) from);
      return;
    }
  } else if (from instanceof FloatWritable) {
    if (to instanceof FloatWritable) {
      ((FloatWritable) to).set(((FloatWritable) from).get());
      return;
    }
  } else if (from instanceof Text) {
    if (to instanceof Text) {
      ((Text) to).set((Text) from);
      return;
    }
  } else if (from instanceof DateWritable) {
    if (to instanceof DateWritable) {
      ((DateWritable) to).set(((DateWritable) from).get());
      return;
    }
  } else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
    ((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
    return;
  } else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
    ((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
    return;
  } else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
    ((BooleanWritable) to).set(((BooleanWritable) from).get());
    return;
  }
  throw new UnsupportedOperationException(String
      .format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
          from.getClass(), to.getClass()));
}
 
Example #11
Source File: OrcTestUtils.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
/**
 * Fill in value in OrcStruct with given schema, assuming {@param w} contains the same schema as {@param schema}.
 * {@param schema} is still necessary to given given {@param w} do contains schema information itself, because the
 * actual value type is only available in {@link TypeDescription} but not {@link org.apache.orc.mapred.OrcValue}.
 *
 * For simplicity here are some assumptions:
 * - We only give 3 primitive values and use them to construct compound values. To make it work for different types that
 * can be widened or shrunk to each other, please use value within small range.
 * - For List, Map or Union, make sure there's at least one entry within the record-container.
 * you may want to try createValueRecursively(TypeDescription) instead of {@link OrcStruct#createValue(TypeDescription)}
 */
public static void fillOrcStructWithFixedValue(WritableComparable w, TypeDescription schema, int unionTag,
    int intValue, String stringValue, boolean booleanValue) {
  switch (schema.getCategory()) {
    case BOOLEAN:
      ((BooleanWritable) w).set(booleanValue);
      break;
    case BYTE:
      ((ByteWritable) w).set((byte) intValue);
      break;
    case SHORT:
      ((ShortWritable) w).set((short) intValue);
      break;
    case INT:
      ((IntWritable) w).set(intValue);
      break;
    case LONG:
      ((LongWritable) w).set(intValue);
      break;
    case FLOAT:
      ((FloatWritable) w).set(intValue * 1.0f);
      break;
    case DOUBLE:
      ((DoubleWritable) w).set(intValue * 1.0);
      break;
    case STRING:
    case CHAR:
    case VARCHAR:
      ((Text) w).set(stringValue);
      break;
    case BINARY:
      throw new UnsupportedOperationException("Binary type is not supported in random orc data filler");
    case DECIMAL:
      throw new UnsupportedOperationException("Decimal type is not supported in random orc data filler");
    case DATE:
    case TIMESTAMP:
    case TIMESTAMP_INSTANT:
      throw new UnsupportedOperationException(
          "Timestamp and its derived types is not supported in random orc data filler");
    case LIST:
      OrcList castedList = (OrcList) w;
      // Here it is not trivial to create typed-object in element-type. So this method expect the value container
      // to at least contain one element, or the traversing within the list will be skipped.
      for (Object i : castedList) {
        fillOrcStructWithFixedValue((WritableComparable) i, schema.getChildren().get(0), unionTag, intValue,
            stringValue, booleanValue);
      }
      break;
    case MAP:
      OrcMap castedMap = (OrcMap) w;
      for (Object entry : castedMap.entrySet()) {
        Map.Entry<WritableComparable, WritableComparable> castedEntry =
            (Map.Entry<WritableComparable, WritableComparable>) entry;
        fillOrcStructWithFixedValue(castedEntry.getKey(), schema.getChildren().get(0), unionTag, intValue,
            stringValue, booleanValue);
        fillOrcStructWithFixedValue(castedEntry.getValue(), schema.getChildren().get(1), unionTag, intValue,
            stringValue, booleanValue);
      }
      break;
    case STRUCT:
      OrcStruct castedStruct = (OrcStruct) w;
      int fieldIdx = 0;
      for (TypeDescription child : schema.getChildren()) {
        fillOrcStructWithFixedValue(castedStruct.getFieldValue(fieldIdx), child, unionTag, intValue, stringValue,
            booleanValue);
        fieldIdx += 1;
      }
      break;
    case UNION:
      OrcUnion castedUnion = (OrcUnion) w;
      TypeDescription targetMemberSchema = schema.getChildren().get(unionTag);
      castedUnion.set(unionTag, OrcUtils.createValueRecursively(targetMemberSchema));
      fillOrcStructWithFixedValue((WritableComparable) castedUnion.getObject(), targetMemberSchema, unionTag,
          intValue, stringValue, booleanValue);
      break;
    default:
      throw new IllegalArgumentException("Unknown type " + schema.toString());
  }
}
 
Example #12
Source File: IIDistinctColumnsJob.java    From Kylin with Apache License 2.0 4 votes vote down vote up
private void setupMapper() throws IOException {

        String tableName = job.getConfiguration().get(BatchConstants.TABLE_NAME);
        String[] dbTableNames = HadoopUtil.parseHiveTableName(tableName);

        log.info("setting hcat input format, db name {} , table name {}", dbTableNames[0],dbTableNames[1]);

        HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);

        job.setInputFormatClass(HCatInputFormat.class);

        job.setMapperClass(IIDistinctColumnsMapper.class);
        job.setCombinerClass(IIDistinctColumnsCombiner.class);
        job.setMapOutputKeyClass(ShortWritable.class);
        job.setMapOutputValueClass(Text.class);
    }