org.apache.reef.io.serialization.Codec Java Examples

@Inject
public ScatterReceiver(@Parameter(CommunicationGroupName.class) final String groupName,
                       @Parameter(OperatorName.class) final String operName,
                       @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                       @Parameter(DataCodec.class) final Codec<T> dataCodec,
                       @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                       @Parameter(TaskVersion.class) final int version,
                       final CommGroupNetworkHandler commGroupNetworkHandler,
                       final NetworkService<GroupCommunicationMessage> netService,
                       final CommunicationGroupServiceClient commGroupClient,
                       final ScatterDecoder scatterDecoder) {
  LOG.finest(operName + "has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.version = version;
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.scatterDecoder = scatterDecoder;
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName,
                                           selfId, driverId, new Sender(netService), version);
  this.commGroupClient = commGroupClient;
  commGroupNetworkHandler.register(this.operName, this);
}
 

@Inject
public BroadcastSender(@Parameter(CommunicationGroupName.class) final String groupName,
                       @Parameter(OperatorName.class) final String operName,
                       @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                       @Parameter(DataCodec.class) final Codec<T> dataCodec,
                       @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                       @Parameter(TaskVersion.class) final int version,
                       final CommGroupNetworkHandler commGroupNetworkHandler,
                       final NetworkService<GroupCommunicationMessage> netService,
                       final CommunicationGroupServiceClient commGroupClient) {
  super();
  this.version = version;
  LOG.finest(operName + "has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.commGroupNetworkHandler = commGroupNetworkHandler;
  this.netService = netService;
  this.sender = new Sender(this.netService);
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName, selfId, driverId, sender, version);
  this.commGroupNetworkHandler.register(this.operName, this);
  this.commGroupClient = commGroupClient;
}
 

private void processTopologyChanges(final GroupCommunicationMessage msg) {
  final String dstId = msg.getSrcid();
  boolean hasTopologyChanged = false;
  LOG.finest(getQualifiedName() + "Checking which nodes need to be updated");
  for (final TaskNode node : nodes.values()) {
    if (!node.isRunning() || node.hasChanges()) {
      hasTopologyChanged = true;
      break;
    }
  }
  final GroupChanges changes = new GroupChangesImpl(hasTopologyChanged);
  final Codec<GroupChanges> changesCodec = new GroupChangesCodec();
  senderStage.onNext(Utils.bldVersionedGCM(groupName, operName,
      ReefNetworkGroupCommProtos.GroupCommMessage.Type.TopologyChanges, driverId, 0, dstId, getNodeVersion(dstId),
      changesCodec.encode(changes)));
}
 

private void onTopologyChanges(final GroupCommunicationMessage msg) {
  LOG.entering("TreeTopology", "onTopologyChanges", new Object[]{getQualifiedName(), msg});
  LOG.fine(getQualifiedName() + "Check TopologyChanges");
  final String dstId = msg.getSrcid();
  boolean hasTopologyChanged = false;
  LOG.finest(getQualifiedName() + "Checking which nodes need to be updated");
  for (final TaskNode node : nodes.values()) {
    if (!node.isRunning() || node.hasChanges()) {
      hasTopologyChanged = true;
      break;
    }
  }
  final GroupChanges changes = new GroupChangesImpl(hasTopologyChanged);
  final Codec<GroupChanges> changesCodec = new GroupChangesCodec();
  LOG.fine(getQualifiedName() + "TopologyChanges: " + changes);
  senderStage.onNext(Utils.bldVersionedGCM(groupName, operName,
      ReefNetworkGroupCommProtos.GroupCommMessage.Type.TopologyChanges, driverId, 0, dstId, getNodeVersion(dstId),
      changesCodec.encode(changes)));
  LOG.exiting("TreeTopology", "onTopologyChanges", getQualifiedName() + msg);
}
 

@Inject
public ScatterSender(@Parameter(CommunicationGroupName.class) final String groupName,
                     @Parameter(OperatorName.class) final String operName,
                     @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                     @Parameter(DataCodec.class) final Codec<T> dataCodec,
                     @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                     @Parameter(TaskVersion.class) final int version,
                     final CommGroupNetworkHandler commGroupNetworkHandler,
                     final NetworkService<GroupCommunicationMessage> netService,
                     final CommunicationGroupServiceClient commGroupClient,
                     final ScatterEncoder scatterEncoder) {
  LOG.finest(operName + "has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.version = version;
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.scatterEncoder = scatterEncoder;
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName,
                                           selfId, driverId, new Sender(netService), version);
  this.commGroupClient = commGroupClient;
  commGroupNetworkHandler.register(this.operName, this);
}
 

@Inject
public BroadcastReceiver(@Parameter(CommunicationGroupName.class) final String groupName,
                         @Parameter(OperatorName.class) final String operName,
                         @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                         @Parameter(DataCodec.class) final Codec<T> dataCodec,
                         @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                         @Parameter(TaskVersion.class) final int version,
                         final CommGroupNetworkHandler commGroupNetworkHandler,
                         final NetworkService<GroupCommunicationMessage> netService,
                         final CommunicationGroupServiceClient commGroupClient) {
  super();
  this.version = version;
  LOG.finest(operName + " has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.commGroupNetworkHandler = commGroupNetworkHandler;
  this.netService = netService;
  this.sender = new Sender(this.netService);
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName, selfId, driverId, sender, version);
  this.commGroupNetworkHandler.register(this.operName, this);
  this.commGroupClient = commGroupClient;
}
 

@Inject
public GatherSender(@Parameter(CommunicationGroupName.class) final String groupName,
                    @Parameter(OperatorName.class) final String operName,
                    @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                    @Parameter(DataCodec.class) final Codec<T> dataCodec,
                    @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                    @Parameter(TaskVersion.class) final int version,
                    final CommGroupNetworkHandler commGroupNetworkHandler,
                    final NetworkService<GroupCommunicationMessage> netService,
                    final CommunicationGroupServiceClient commGroupClient) {
  LOG.finest(operName + "has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.version = version;
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.netService = netService;
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName,
                                           selfId, driverId, new Sender(netService), version);
  this.commGroupClient = commGroupClient;
  commGroupNetworkHandler.register(this.operName, this);
}
 

@Inject
public ReduceReceiver(@Parameter(CommunicationGroupName.class) final String groupName,
                      @Parameter(OperatorName.class) final String operName,
                      @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                      @Parameter(DataCodec.class) final Codec<T> dataCodec,
                      @Parameter(ReduceFunctionParam.class) final ReduceFunction<T> reduceFunction,
                      @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                      @Parameter(TaskVersion.class) final int version,
                      final CommGroupNetworkHandler commGroupNetworkHandler,
                      final NetworkService<GroupCommunicationMessage> netService,
                      final CommunicationGroupServiceClient commGroupClient) {
  super();
  this.version = version;
  LOG.finest(operName + " has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.reduceFunction = reduceFunction;
  this.commGroupNetworkHandler = commGroupNetworkHandler;
  this.netService = netService;
  this.sender = new Sender(this.netService);
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName, selfId, driverId, sender, version);
  this.commGroupNetworkHandler.register(this.operName, this);
  this.commGroupClient = commGroupClient;
}
 

@Inject
public GatherReceiver(@Parameter(CommunicationGroupName.class) final String groupName,
                      @Parameter(OperatorName.class) final String operName,
                      @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
                      @Parameter(DataCodec.class) final Codec<T> dataCodec,
                      @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                      @Parameter(TaskVersion.class) final int version,
                      final CommGroupNetworkHandler commGroupNetworkHandler,
                      final NetworkService<GroupCommunicationMessage> netService,
                      final CommunicationGroupServiceClient commGroupClient) {
  LOG.finest(operName + " has CommGroupHandler-" + commGroupNetworkHandler.toString());
  this.version = version;
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName,
                                           selfId, driverId, new Sender(netService), version);
  this.commGroupClient = commGroupClient;
  commGroupNetworkHandler.register(this.operName, this);
}
 

public ReduceOperatorSpec(final String receiverId,
                          final Class<? extends Codec> dataCodecClass,
                          final Class<? extends ReduceFunction> redFuncClass) {
  super();
  this.receiverId = receiverId;
  this.dataCodecClass = dataCodecClass;
  this.redFuncClass = redFuncClass;
}
 

@Override
public <T> T recvFromChildren(final Reduce.ReduceFunction<T> redFunc, final Codec<T> dataCodec)
    throws ParentDeadException {
  LOG.entering("OperatorTopologyImpl", "recvFromChildren", getQualifiedName());
  refreshEffectiveTopology();
  assert effectiveTopology != null;
  final T retVal = effectiveTopology.recvFromChildren(redFunc, dataCodec);
  LOG.exiting("OperatorTopologyImpl", "recvFromChildren", getQualifiedName());
  return retVal;
}
 

@Inject
public ReduceSender(
    @Parameter(CommunicationGroupName.class) final String groupName,
    @Parameter(OperatorName.class) final String operName,
    @Parameter(TaskConfigurationOptions.Identifier.class) final String selfId,
    @Parameter(DataCodec.class) final Codec<T> dataCodec,
    @Parameter(ReduceFunctionParam.class) final ReduceFunction<T> reduceFunction,
    @Parameter(DriverIdentifierGroupComm.class) final String driverId,
    @Parameter(TaskVersion.class) final int version,
    final CommGroupNetworkHandler commGroupNetworkHandler,
    final NetworkService<GroupCommunicationMessage> netService,
    final CommunicationGroupServiceClient commGroupClient) {

  super();

  LOG.log(Level.FINEST, "{0} has CommGroupHandler-{1}",
      new Object[]{operName, commGroupNetworkHandler});

  this.version = version;
  this.groupName = Utils.getClass(groupName);
  this.operName = Utils.getClass(operName);
  this.dataCodec = dataCodec;
  this.reduceFunction = reduceFunction;
  this.commGroupNetworkHandler = commGroupNetworkHandler;
  this.netService = netService;
  this.sender = new Sender(this.netService);
  this.topology = new OperatorTopologyImpl(this.groupName, this.operName, selfId, driverId, sender, version);
  this.commGroupNetworkHandler.register(this.operName, this);
  this.commGroupClient = commGroupClient;
}
 

/**
 * Encode elements into byte arrays, and distribute them across Tasks indicated by Identifiers.
 * Note that elements are distributed in the exact order specified in
 * {@code elements} and not in a round-robin fashion.
 * For example, (1, 2, 3, 4) uniformly distributed to (task1, task2, task3) would be
 * {task1: (1, 2), task2: (3), task3: (4)}.
 *
 * @param elements list of data elements to encode
 * @param counts list of numbers specifying how many elements each Task should receive
 * @param taskOrder list of Identifiers indicating Task Ids
 * @param codec class for encoding data
 * @param <T> type of data
 * @return byte representation of a map of identifiers to encoded data
 */
private <T> Map<String, byte[]> encodeAndDistributeElements(final List<T> elements,
                                                            final List<Integer> counts,
                                                            final List<? extends Identifier> taskOrder,
                                                            final Codec<T> codec) {
  final Map<String, byte[]> taskIdToBytes = new HashMap<>();

  int elementsIndex = 0;
  for (int taskOrderIndex = 0; taskOrderIndex < taskOrder.size(); taskOrderIndex++) {
    final int elementCount = counts.get(taskOrderIndex);

    try (ByteArrayOutputStream bstream = new ByteArrayOutputStream();
         DataOutputStream dstream = new DataOutputStream(bstream)) {

      dstream.writeInt(elementCount);
      for (final T element : elements.subList(elementsIndex, elementsIndex + elementCount)) {
        final byte[] encodedElement = codec.encode(element);
        dstream.writeInt(encodedElement.length);
        dstream.write(encodedElement);
      }
      taskIdToBytes.put(taskOrder.get(taskOrderIndex).toString(), bstream.toByteArray());

    } catch (final IOException e) {
      throw new RuntimeException("IOException",  e);
    }

    elementsIndex += elementCount;
  }

  return taskIdToBytes;
}
 

public <T> Map<String, byte[]> encode(final List<T> elements,
                                      final List<Integer> counts,
                                      final List<? extends Identifier> taskOrder,
                                      final Codec<T> dataCodec) {

  // first assign data to all tasks
  final Map<String, byte[]> taskIdToBytes = encodeAndDistributeElements(elements, counts, taskOrder, dataCodec);
  // then organize the data so that a node keeps its own data as well as its descendants' data
  final Map<String, byte[]> childIdToBytes = new HashMap<>();

  for (final TopologySimpleNode node : commGroupClient.getTopologySimpleNodeRoot().getChildren()) {
    childIdToBytes.put(node.getTaskId(), encodeScatterMsgForNode(node, taskIdToBytes));
  }
  return childIdToBytes;
}
 

@Test
public void testInterop() throws ServiceException {
  final LocalStorageService service = new LocalStorageService("spoolTest", "file");
  final Codec<Integer> c = new IntegerCodec();


  final CodecFileAccumulable<Integer, Codec<Integer>> f = new CodecFileAccumulable<>(
      service, c);
  final CodecFileIterable<Integer, Codec<Integer>> g = new CodecFileIterable<>(
      new File(f.getName()), c);
  test(f, g);
  service.getScratchSpace().delete();
}
 

@Test
public void testCodecRamMap() {
  final RamStorageService ramStore = new RamStorageService();
  final Codec<Integer> c = new IntegerCodec();
  final ExternalMap<Integer> m = new CodecRamMap<>(ramStore, c);
  genericTest(m);
}
 

/**
 * Test that {@code ScatterEncoder} and {@code ScatterDecoder} function correctly.
 * Create a small topology of 4 nodes and simulate a scatter operation.
 */
@Test
public void testEncodeDecode() {
  final IdentifierFactory ifac = new StringIdentifierFactory();
  final Codec<Integer> codec = new SerializableCodec<>();

  final List<Integer> elements = new LinkedList<>();
  for (int element = 0; element < 400; element++) {
    elements.add(element);
  }

  final List<Integer> counts = new LinkedList<>();
  final List<Identifier> taskOrder = new LinkedList<>();
  for (int index = 0; index < 4; index++) {
    counts.add(100);
    taskOrder.add(ifac.getNewInstance("Task-" + index));
  }

  final TopologySimpleNode rootNode = new TopologySimpleNode("Task-0");
  final TopologySimpleNode childNode1 = new TopologySimpleNode("Task-1");
  final TopologySimpleNode childNode2 = new TopologySimpleNode("Task-2");
  final TopologySimpleNode childNode3 = new TopologySimpleNode("Task-3");
  rootNode.addChild(childNode1);
  rootNode.addChild(childNode2);
  childNode1.addChild(childNode3);

  final CommunicationGroupServiceClient mockCommGroupClient = mock(CommunicationGroupServiceClient.class);
  when(mockCommGroupClient.getTopologySimpleNodeRoot()).thenReturn(rootNode);
  final ScatterEncoder scatterEncoder = new ScatterEncoder(mockCommGroupClient);
  final ScatterDecoder scatterDecoder = new ScatterDecoder();

  final Map<String, byte[]> encodedDataMap = scatterEncoder.encode(elements, counts, taskOrder, codec);

  // check msg correctness for childNode1 (Task-1)
  final ScatterData childNode1Data = scatterDecoder.decode(encodedDataMap.get(childNode1.getTaskId()));
  for (int index = 0; index < 100; index++) {
    assertTrue(index + 100 == codec.decode(childNode1Data.getMyData()[index]));
  }
  assertTrue(childNode1Data.getChildrenData().containsKey("Task-3"));
  assertEquals(childNode1Data.getChildrenData().size(), 1);

  // check msg correctness for childNode2 (Task-2)
  final ScatterData childNode2Data = scatterDecoder.decode(encodedDataMap.get(childNode2.getTaskId()));
  for (int index = 0; index < 100; index++) {
    assertTrue(index + 200 == codec.decode(childNode2Data.getMyData()[index]));
  }
  assertTrue(childNode2Data.getChildrenData().isEmpty());
}
 

@Inject
public CodecRamMap(final RamStorageService ramStore,
                   @Parameter(RamMapCodec.class) final Codec<T> c) {
  this.c = c;
  this.map = new ConcurrentSkipListMap<CharSequence, byte[]>();
}
 

CodecFileIterator(final Codec<T> codec, final File file) throws IOException {
  this.in = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
  this.codec = codec;
  this.readNextSize();
}
 

CodecFileAccumulator(final Codec<T> codec, final File file) throws IOException {
  this.codec = codec;
  this.out = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
}
 

public Builder setDataCodecClass(final Class<? extends Codec> codecClazz) {
  this.dataCodecClass = codecClazz;
  return this;
}
 

@Override
public Class<? extends Codec> getDataCodecClass() {
  return dataCodecClass;
}
 

public Builder setDataCodecClass(final Class<? extends Codec> dataCodecClass) {
  this.dataCodecClass = dataCodecClass;
  return this;
}
 

public Builder setDataCodecClass(final Class<? extends Codec> dataCodecClass) {
  this.dataCodecClass = dataCodecClass;
  return this;
}
 

@Override
public Class<? extends Codec> getDataCodecClass() {
  return dataCodecClass;
}
 

public ScatterOperatorSpec(final String senderId,
                           final Class<? extends Codec> dataCodecClass) {
  this.senderId = senderId;
  this.dataCodecClass = dataCodecClass;
}
 

public Builder setDataCodecClass(final Class<? extends Codec> codecClazz) {
  this.dataCodecClass = codecClazz;
  return this;
}
 

@Override
public Class<? extends Codec> getDataCodecClass() {
  return dataCodecClass;
}
 

public BroadcastOperatorSpec(final String senderId,
                             final Class<? extends Codec> dataCodecClass) {
  super();
  this.senderId = senderId;
  this.dataCodecClass = dataCodecClass;
}
 

@Override
public Class<? extends Codec> getDataCodecClass() {
  return dataCodecClass;
}