Java Code Examples for org.apache.hadoop.io.serializer.Serialization

The following examples show how to use org.apache.hadoop.io.serializer.Serialization. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: Chain.java    License: Apache License 2.0 6 votes vote down vote up
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
 
Example 2
Source Project: big-c   Source File: Chain.java    License: Apache License 2.0 6 votes vote down vote up
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
 
Example 3
Source Project: ignite   Source File: HadoopV2TaskContext.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets serializer for specified class.
 *
 * @param cls Class.
 * @param jobConf Job configuration.
 * @return Appropriate serializer.
 */
@SuppressWarnings("unchecked")
private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf) throws IgniteCheckedException {
    A.notNull(cls, "cls");

    SerializationFactory factory = new SerializationFactory(jobConf);

    Serialization<?> serialization = factory.getSerialization(cls);

    if (serialization == null)
        throw new IgniteCheckedException("Failed to find serialization for: " + cls.getName());

    if (serialization.getClass() == WritableSerialization.class)
        return new HadoopWritableSerialization((Class<? extends Writable>)cls);

    return new HadoopSerializationWrapper(serialization, cls);
}
 
Example 4
Source Project: RDFS   Source File: Chain.java    License: Apache License 2.0 6 votes vote down vote up
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
 
Example 5
Source Project: hadoop-gpu   Source File: Chain.java    License: Apache License 2.0 6 votes vote down vote up
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
 
Example 6
Source Project: hadoop   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the OutputCollector to be used by a Mapper instance in the chain.
 *
 * @param mapperIndex index of the Mapper instance to get the OutputCollector.
 * @param output      the original OutputCollector of the task.
 * @param reporter    the reporter of the task.
 * @return the OutputCollector to be used in the chain.
 */
@SuppressWarnings({"unchecked"})
public OutputCollector getMapperCollector(int mapperIndex,
                                          OutputCollector output,
                                          Reporter reporter) {
  Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
  Serialization valueSerialization =
    mappersValueSerialization.get(mapperIndex);
  return new ChainOutputCollector(mapperIndex, keySerialization,
                                  valueSerialization, output, reporter);
}
 
Example 7
Source Project: hadoop   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(int index, Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = index + 1;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 8
Source Project: hadoop   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = 0;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 9
Source Project: hadoop   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
public void collect(K key, V value) throws IOException {
  if (nextMapperIndex < mappers.size()) {
    // there is a next mapper in chain

    // only need to ser/deser if there is next mapper in the chain
    if (keySerialization != null) {
      key = makeCopyForPassByValue(keySerialization, key);
      value = makeCopyForPassByValue(valueSerialization, value);
    }

    // gets ser/deser and mapper of next in chain
    Serialization nextKeySerialization =
      mappersKeySerialization.get(nextMapperIndex);
    Serialization nextValueSerialization =
      mappersValueSerialization.get(nextMapperIndex);
    Mapper nextMapper = mappers.get(nextMapperIndex);

    // invokes next mapper in chain
    nextMapper.map(key, value,
                   new ChainOutputCollector(nextMapperIndex,
                                            nextKeySerialization,
                                            nextValueSerialization,
                                            output, reporter),
                   reporter);
  } else {
    // end of chain, user real output collector
    output.collect(key, value);
  }
}
 
Example 10
Source Project: big-c   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the OutputCollector to be used by a Mapper instance in the chain.
 *
 * @param mapperIndex index of the Mapper instance to get the OutputCollector.
 * @param output      the original OutputCollector of the task.
 * @param reporter    the reporter of the task.
 * @return the OutputCollector to be used in the chain.
 */
@SuppressWarnings({"unchecked"})
public OutputCollector getMapperCollector(int mapperIndex,
                                          OutputCollector output,
                                          Reporter reporter) {
  Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
  Serialization valueSerialization =
    mappersValueSerialization.get(mapperIndex);
  return new ChainOutputCollector(mapperIndex, keySerialization,
                                  valueSerialization, output, reporter);
}
 
Example 11
Source Project: big-c   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(int index, Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = index + 1;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 12
Source Project: big-c   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = 0;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 13
Source Project: big-c   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
public void collect(K key, V value) throws IOException {
  if (nextMapperIndex < mappers.size()) {
    // there is a next mapper in chain

    // only need to ser/deser if there is next mapper in the chain
    if (keySerialization != null) {
      key = makeCopyForPassByValue(keySerialization, key);
      value = makeCopyForPassByValue(valueSerialization, value);
    }

    // gets ser/deser and mapper of next in chain
    Serialization nextKeySerialization =
      mappersKeySerialization.get(nextMapperIndex);
    Serialization nextValueSerialization =
      mappersValueSerialization.get(nextMapperIndex);
    Mapper nextMapper = mappers.get(nextMapperIndex);

    // invokes next mapper in chain
    nextMapper.map(key, value,
                   new ChainOutputCollector(nextMapperIndex,
                                            nextKeySerialization,
                                            nextValueSerialization,
                                            output, reporter),
                   reporter);
  } else {
    // end of chain, user real output collector
    output.collect(key, value);
  }
}
 
Example 14
Source Project: ignite   Source File: HadoopSerializationWrapper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param serialization External serializer to wrap.
 * @param cls The class to serialize.
 */
public HadoopSerializationWrapper(Serialization<T> serialization, Class<T> cls) throws IgniteCheckedException {
    assert cls != null;

    serializer = serialization.getSerializer(cls);
    deserializer = serialization.getDeserializer(cls);

    try {
        serializer.open(outStream);
        deserializer.open(inStream);
    }
    catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}
 
Example 15
Source Project: RDFS   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the OutputCollector to be used by a Mapper instance in the chain.
 *
 * @param mapperIndex index of the Mapper instance to get the OutputCollector.
 * @param output      the original OutputCollector of the task.
 * @param reporter    the reporter of the task.
 * @return the OutputCollector to be used in the chain.
 */
@SuppressWarnings({"unchecked"})
public OutputCollector getMapperCollector(int mapperIndex,
                                          OutputCollector output,
                                          Reporter reporter) {
  Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
  Serialization valueSerialization =
    mappersValueSerialization.get(mapperIndex);
  return new ChainOutputCollector(mapperIndex, keySerialization,
                                  valueSerialization, output, reporter);
}
 
Example 16
Source Project: RDFS   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(int index, Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = index + 1;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 17
Source Project: RDFS   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = 0;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 18
Source Project: RDFS   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
public void collect(K key, V value) throws IOException {
  if (nextMapperIndex < mappers.size()) {
    // there is a next mapper in chain

    // only need to ser/deser if there is next mapper in the chain
    if (keySerialization != null) {
      key = makeCopyForPassByValue(keySerialization, key);
      value = makeCopyForPassByValue(valueSerialization, value);
    }

    // gets ser/deser and mapper of next in chain
    Serialization nextKeySerialization =
      mappersKeySerialization.get(nextMapperIndex);
    Serialization nextValueSerialization =
      mappersValueSerialization.get(nextMapperIndex);
    Mapper nextMapper = mappers.get(nextMapperIndex);

    // invokes next mapper in chain
    nextMapper.map(key, value,
                   new ChainOutputCollector(nextMapperIndex,
                                            nextKeySerialization,
                                            nextValueSerialization,
                                            output, reporter),
                   reporter);
  } else {
    // end of chain, user real output collector
    output.collect(key, value);
  }
}
 
Example 19
Source Project: hadoop-gpu   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the OutputCollector to be used by a Mapper instance in the chain.
 *
 * @param mapperIndex index of the Mapper instance to get the OutputCollector.
 * @param output      the original OutputCollector of the task.
 * @param reporter    the reporter of the task.
 * @return the OutputCollector to be used in the chain.
 */
@SuppressWarnings({"unchecked"})
public OutputCollector getMapperCollector(int mapperIndex,
                                          OutputCollector output,
                                          Reporter reporter) {
  Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
  Serialization valueSerialization =
    mappersValueSerialization.get(mapperIndex);
  return new ChainOutputCollector(mapperIndex, keySerialization,
                                  valueSerialization, output, reporter);
}
 
Example 20
Source Project: hadoop-gpu   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(int index, Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = index + 1;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 21
Source Project: hadoop-gpu   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
public ChainOutputCollector(Serialization<K> keySerialization,
                            Serialization<V> valueSerialization,
                            OutputCollector output, Reporter reporter) {
  this.nextMapperIndex = 0;
  this.keySerialization = keySerialization;
  this.valueSerialization = valueSerialization;
  this.output = output;
  this.reporter = reporter;
}
 
Example 22
Source Project: hadoop-gpu   Source File: Chain.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
public void collect(K key, V value) throws IOException {
  if (nextMapperIndex < mappers.size()) {
    // there is a next mapper in chain

    // only need to ser/deser if there is next mapper in the chain
    if (keySerialization != null) {
      key = makeCopyForPassByValue(keySerialization, key);
      value = makeCopyForPassByValue(valueSerialization, value);
    }

    // gets ser/deser and mapper of next in chain
    Serialization nextKeySerialization =
      mappersKeySerialization.get(nextMapperIndex);
    Serialization nextValueSerialization =
      mappersValueSerialization.get(nextMapperIndex);
    Mapper nextMapper = mappers.get(nextMapperIndex);

    // invokes next mapper in chain
    nextMapper.map(key, value,
                   new ChainOutputCollector(nextMapperIndex,
                                            nextKeySerialization,
                                            nextValueSerialization,
                                            output, reporter),
                   reporter);
  } else {
    // end of chain, user real output collector
    output.collect(key, value);
  }
}