/**
*
* Copyright (c) 2017 ytk-mp4j https://github.com/yuantiku
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package com.fenbi.mp4j.comm;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.fenbi.mp4j.exception.Mp4jException;
import com.fenbi.mp4j.meta.ArrayMetaData;
import com.fenbi.mp4j.meta.MapMetaData;
import com.fenbi.mp4j.meta.MetaData;
import com.fenbi.mp4j.operand.*;
import com.fenbi.mp4j.operator.*;
import com.fenbi.mp4j.rpc.IServer;
import com.fenbi.mp4j.rpc.Server;
import com.fenbi.mp4j.utils.CommUtils;
import com.fenbi.mp4j.utils.KryoUtils;
import com.fenbi.mp4j.utils.ScatterAllocate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.ArrayPrimitiveWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.lang.management.ManagementFactory;
import java.lang.reflect.Array;
import java.net.*;
import java.util.*;
import java.util.concurrent.*;
/**
* ProcessCommSlave is used as multi-processes communication.
* @author xialong
*/
public class ProcessCommSlave {
public static final Logger LOG = LoggerFactory.getLogger(ProcessCommSlave.class);
public final static String HOST_PORT_DELIM = "###";
public final static int SOCK_CONN_RETRY_TIME = 50;
public final static int SOCK_CONN_SLEEP_TIME = 30;
private final int slaveNum;
private final int rank;
private final String rankMsgPrefix;
private final String[] slaveHosts;
private final int[] slavePorts;
private volatile Socket sendSockRef;
private volatile Socket recvSockRef;
private final ServerSocket recvDataSock;
private volatile boolean closed = false;
private volatile int curStep = 0;
private final ScheduledExecutorService scheduledThreadPool;
private volatile int heartbeatExceptionNum = 0;
private final IServer server;
private volatile Operand operand;
private BlockingDeque<MetaData> sendQueue = new LinkedBlockingDeque<>();
private BlockingDeque<MetaData> recvQueue = new LinkedBlockingDeque<>();
private BlockingDeque<MetaData> recvResultQueue = new LinkedBlockingDeque<>();
private final Thread sendDataTask = new Thread() {
@Override
public void run() {
try {
while (true) {
MetaData metaData = sendQueue.take();
Output output = getOutput(metaData.getDestRank());
operand.send(output, metaData);
}
} catch (Exception e) {
try {
exception(e);
} catch (Mp4jException e1) {
LOG.error("Mp4jException", e);
}
}
}
};
private final Thread recvDataTask = new Thread() {
@Override
public void run() {
try {
while (true) {
MetaData metaData = recvQueue.take();
MetaData recvMetaData = operand.recv(getInput(), metaData);
recvResultQueue.put(recvMetaData);
}
} catch (Exception e) {
try {
exception(e);
} catch (Mp4jException e1) {
LOG.error("Mp4jException", e);
}
}
}
};
private final Thread sendHeartBeatTask = new Thread() {
@Override
public void run() {
heartbeat();
}
};
/**
* Process communication constructor, every process have just only one ProcessCommSlave instance.
* @param loginName if you use ssh to execute command, you must provide login name, e.g. ssh loginName@host "your command"
* @param masterHost master host name
* @param masterPort master host port
* @throws Mp4jException
*/
public ProcessCommSlave(String loginName,
String masterHost,
int masterPort) throws Mp4jException {
try {
LOG.info("master host:" + masterHost + ", master port:" + masterPort);
InetSocketAddress address = new InetSocketAddress(masterHost, masterPort);
server = (IServer) RPC.getProxy(IServer.class, IServer.versionID, address, new Configuration());
String host = InetAddress.getLocalHost().getHostName();
recvDataSock = new ServerSocket(0);
int port = recvDataSock.getLocalPort();
Text addresses = server.getAllSlavesInfo(new Text(host + HOST_PORT_DELIM + port));
if (address == null) {
throw new Mp4jException("slaves connecting master failed, may be this slave restarted or master port is occupied, task failed!");
}
String[] slavesAddresses = addresses.toString().split(Server.ADDRESS_DELIM);
slaveNum = slavesAddresses.length - 1;
rank = Integer.parseInt(slavesAddresses[slaveNum]);
rankMsgPrefix = "[rank=" + rank + "] ";
LOG.info("this slave recv data port:" + port);
LOG.info("slave num:" + slaveNum);
LOG.info("slave rank:" + rank);
// get pid
// get name representing the running Java virtual machine.
String name = ManagementFactory.getRuntimeMXBean().getName();
String pid = name.split("@")[0];
LOG.info("Pid is:" + pid);
if (slaveNum > 1) {
server.killMe(rank, new Text("ssh " + loginName + "@" + host +
" \"kill -9 " + pid + "\""));
} else {
server.killMe(rank, new Text("kill -9 " + pid));
}
slaveHosts = new String[slaveNum];
slavePorts = new int[slaveNum];
for (int i = 0; i < slaveNum; i++) {
String[] addr = slavesAddresses[i].split(HOST_PORT_DELIM);
slaveHosts[i] = addr[0];
slavePorts[i] = Integer.parseInt(addr[1]);
}
LOG.info("slaves addresses:");
for (int i = 0; i < slaveNum; i++) {
LOG.info(slaveHosts[i] + ":" + slavePorts[i]);
}
this.scheduledThreadPool = Executors.newScheduledThreadPool(1);
scheduledThreadPool.scheduleAtFixedRate(sendHeartBeatTask, 5, 15, TimeUnit.SECONDS);
sendDataTask.start();
recvDataTask.start();
} catch (Exception e) {
LOG.error("exception!", e);
throw new Mp4jException(e.getCause());
}
info("this slave init finished!");
}
private void heartbeat() {
try {
if (!closed) {
server.heartbeat(rank);
}
} catch (Exception e) {
e.printStackTrace();
heartbeatExceptionNum ++;
LOG.error("rank:" + rank + " send heartbeat exception, exception time:" + heartbeatExceptionNum, e);
if (heartbeatExceptionNum > 4) {
LOG.error("heart beat exception > 4 master may be shutdowned! this slave will be shutdowned...");
System.exit(4);
}
}
}
/**
* close communication.
* @param code close code.
* @throws Mp4jException
*/
public void close(int code) throws Mp4jException {
LOG.info("close code=" + code);
if (closed)
return;
if (server != null) {
try {
server.close(rank, code);
} catch (Exception e) {
throw new Mp4jException("Exception in close!", e.getCause());
}
LOG.info("reduce closed!");
}
try {
if (recvSockRef != null) {
recvSockRef.close();
LOG.info("Recv operand socket closed!");
}
if (recvDataSock != null) {
recvDataSock.close();
LOG.info("Data server socket closed!");
}
if (sendSockRef != null) {
sendSockRef.close();
LOG.info("Send operand socket closed!");
}
} catch (IOException e) {
throw new Mp4jException("exception in slave close!", e.getCause());
}
this.operand = null;
closed = true;
LOG.info("reduce closed!");
}
/**
* create a file in master
* @param content content in a file
* @param fileName file name to be write
* @throws Exception
*/
public void writeFile(String content , String fileName) throws Exception {
server.writeFile(new Text(content), new Text(fileName));
}
/**
* send information to master
* @param info information
* @param onlyRank0 if just rank 0 can send information successfully.
* @throws Mp4jException
*/
public void info(String info, boolean onlyRank0) throws Mp4jException {
try {
if (!onlyRank0) {
server.info(rank, new Text(rankMsgPrefix + info));
} else {
if (rank == 0) {
server.info(rank, new Text(info));
}
}
} catch (Exception e) {
throw new Mp4jException("exception in slave report!", e.getCause());
}
}
/**
* only rank 0 can send information successfully.
* @param info information
* @throws Mp4jException
*/
public void info(String info) throws Mp4jException {
info(info, true);
}
/**
* send debug information to master
* @param debug debug information
* @param onlyRank0 if just rank 0 can send debug information successfully.
* @throws Mp4jException
*/
public void debug(String debug, boolean onlyRank0) throws Mp4jException {
try {
if (!onlyRank0) {
server.debug(rank, new Text(rankMsgPrefix + debug));
} else {
if (rank == 0) {
server.debug(rank, new Text(debug));
}
}
} catch (Exception e) {
throw new Mp4jException("exception in slave report!", e.getCause());
}
}
/**
*
* only rank 0 can send debug information successfully.
* @param debug debug information
* @throws Mp4jException
*/
public void debug(String debug) throws Mp4jException {
debug(debug, true);
}
/**
* send error information to master.
* @param error error info
* @throws Mp4jException
*/
public void error(String error) throws Mp4jException {
try {
server.error(rank, new Text(rankMsgPrefix + error));
} catch (Exception e) {
throw new Mp4jException("exception in slave report!", e.getCause());
}
}
/**
* send exception information to master.
* @param e exception
* @throws Mp4jException
*/
public void exception(Exception e) throws Mp4jException {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
try {
error("slave exception:" + sw.toString());
Thread.sleep(5000);
close(1);
} catch (Exception e1) {
throw new Mp4jException("exception in slave report!", e1.getCause());
}
}
/**
* slave number
* @return
*/
public int getSlaveNum() {
return slaveNum;
}
/**
* the rank of this process
* @return
*/
public int getRank() {
return rank;
}
private Socket getSendDataSocket(int targetRank)
throws InterruptedException, IOException {
Socket sendDataSock = null;
LOG.debug("send data, target rank:" + targetRank);
for (int i = 0; i < SOCK_CONN_RETRY_TIME; i++) {
try {
String targetAddress = slaveHosts[targetRank];
int targetRecvDataPort = slavePorts[targetRank];
sendDataSock = new Socket(targetAddress, targetRecvDataPort);
sendDataSock.setTcpNoDelay(true);
sendDataSock.setSoLinger(true, 160);
break;
} catch (UnknownHostException e) {
Thread.sleep(SOCK_CONN_SLEEP_TIME);
if (i == SOCK_CONN_RETRY_TIME - 1)
throw e;
continue;
} catch (IOException e) {
Thread.sleep(SOCK_CONN_SLEEP_TIME);
if (i == SOCK_CONN_RETRY_TIME - 1)
throw e;
continue;
}
}
sendSockRef = sendDataSock;
return sendDataSock;
}
private Socket getRecvDataSocket() throws IOException {
Socket recvDataSock = this.recvDataSock.accept();
LOG.debug("recv socket:" + recvDataSock);
recvSockRef = recvDataSock;
return recvDataSock;
}
/**
* synchronizing processes
* @throws Mp4jException
*/
public void barrier() throws Mp4jException {
try {
server.barrier();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
private <T> MetaData dynamicBinaryTreeGather(int rootRank, MetaData thisMetaData) throws Mp4jException {
try {
LOG.info("thismetadata:" + thisMetaData);
int thatRank = server.exchange(rank);
if ((rank < thatRank ||
rank == rootRank) &&
thatRank != rootRank) {
LOG.info("this rank:" + rank + " recv data! that rank:" + thatRank);
while (true) {
recvQueue.put(thisMetaData);
MetaData thatMetaData = recvResultQueue.take();
LOG.info("recv thatmetadata:" + thatMetaData);
// merge
switch (operand.getContainer()) {
case ARRAY:
thisMetaData.stepIncr(1);
thisMetaData.setSum(thisMetaData.getSum() + thatMetaData.getSum());
thisMetaData.setSegNum(thisMetaData.getSegNum() + thatMetaData.getSegNum());
ArrayMetaData thatArrayMetaData = thatMetaData.convertToArrayMetaData();
ArrayMetaData thisArrayMetaData = thisMetaData.convertToArrayMetaData();
// append ranks, froms, tos
thisArrayMetaData.getRanks().addAll(thatArrayMetaData.getRanks());
thisArrayMetaData.getSegFroms().addAll(thatArrayMetaData.getSegFroms());
thisArrayMetaData.getSegTos().addAll(thatArrayMetaData.getSegTos());
break;
case MAP:
thisMetaData.stepIncr(1);
thisMetaData.setSum(thisMetaData.getSum() + thatMetaData.getSum());
thisMetaData.setSegNum(1);
MapMetaData thatMapMetaData = thatMetaData.convertToMapMetaData();
MapMetaData thisMapMetaData = thisMetaData.convertToMapMetaData();
thisMapMetaData.getRanks().addAll(thatMapMetaData.getRanks());
// merge map, recv process always has 1 map
List<Map<String, T>> thatMapDataList = thatMetaData.getMapDataList();
Map<String, T> thisMap = (Map<String, T>)thisMapMetaData.getMapDataList().get(0);
for (Map<String, T> thatMap : thatMapDataList) {
for (Map.Entry<String, T> entry : thatMap.entrySet()) {
thisMap.put(entry.getKey(), entry.getValue());
}
}
// reset dataNums = 1
thisMapMetaData.setDataNums(Arrays.asList(thisMap.size()));
break;
default:
throw new Mp4jException("unsupported container:" + operand.getContainer());
}
LOG.info("merged thisMetaData:" + thisMetaData);
if (thisMetaData.getSum() != slaveNum) {
thatRank = server.exchange(rank);
if (!((rank < thatRank ||
rank == rootRank) &&
thatRank != rootRank)) {
break;
}
} else {
if (rank != rootRank) {
throw new Mp4jException("dynamic binary gather must stop at rootRank:" + rootRank + ", thisRank:" + rank);
}
barrier();
LOG.info("root gather finished");
return thisMetaData;
}
}
}
LOG.info("this rank:" + rank + " send data! that rank:" + thatRank);
thisMetaData.setDestRank(thatRank);
LOG.info("send thatmetadata:" + thisMetaData);
sendQueue.put(thisMetaData);
barrier();
LOG.info("gather finished");
return thisMetaData;
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Takes elements from all processes and gathers them to root process, the data container is array,
* process i send data interval [sendfroms[i], sendtos[i]) to root process, placed in the same positions,
* s.t. sendfroms[i] ≤ sendtos[i], sendfroms[i] ≥ sendtos[i-1]
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param sendfroms sending start positions, included.
* @param sendtos sending end positions, excluded
* @param rootRank root rank
* @return if this process is root, returned is gathered elements, otherwise,
* is invalid array, contains intermediate results.
* @throws Mp4jException
*/
public <T> T gatherArray(T arrData, Operand operand, int[] sendfroms, int[] sendtos, int rootRank) throws Mp4jException {
if (sendfroms.length != slaveNum) {
throw new Mp4jException("sendfroms array length must be equal to slaveNum");
}
if (sendtos.length != slaveNum) {
throw new Mp4jException("sendtos array length must be equal to slaveNum");
}
if (slaveNum == 1) {
return arrData;
}
CommUtils.isfromsTosLegal(sendfroms, sendtos);
try {
this.operand = operand;
this.operand.setCollective(Collective.GATHER);
this.operand.setContainer(Container.ARRAY);
ArrayMetaData<T> arrayMetaData = new ArrayMetaData<>();
arrayMetaData.setSrcRank(rank)
.setDestRank(-1)
.setStep(0)
.setSum(1)
.setCollective(Collective.GATHER)
.insert(rank, sendfroms[rank], sendtos[rank]);
arrayMetaData.setArrData(arrData);
return (T)dynamicBinaryTreeGather(rootRank, arrayMetaData).getArrData();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Takes elements from all processes and gathers them to root process, the data container is map,
* the data in map regardless of the order, if different process have same keys, only one key(random) will
* be saved in root process.
* @param mapData key is {@code String}, value is any object
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param rootRank root rank
* @return if this process is root, returned is gathered elements, otherwise,
* is invalid map, contained intermediate comm result or null.
* @throws Mp4jException
*/
public <T> Map<String, T> gatherMap(Map<String, T> mapData, Operand operand, int rootRank) throws Mp4jException {
if (slaveNum == 1) {
return mapData;
}
try {
this.operand = operand;
this.operand.setCollective(Collective.GATHER);
this.operand.setContainer(Container.MAP);
MapMetaData<T> mapMetaData = new MapMetaData<>();
mapMetaData.setSrcRank(rank)
.setDestRank(-1)
.setStep(0)
.setSum(1)
.setCollective(Collective.GATHER)
.insert(rank, mapData.size());
List<Map<String, T>> listMapList = new ArrayList<>();
listMapList.add(mapData);
mapMetaData.setMapDataList(listMapList);
return (Map<String, T>)dynamicBinaryTreeGather(rootRank, mapMetaData).getMapDataList().get(0);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Takes elements from all processes and gathers them to all processes.
* the operation can be viewed as a combination of gather and broadcast.
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param froms the start position of process i to gathered is froms[i]
* @param tos the end position of process i to gathered is tos[i]
* @return allgathered array
* @throws Mp4jException
*/
public <T> T allgatherArray(T arrData, Operand operand, int[] froms, int[] tos) throws Mp4jException {
if (froms.length != slaveNum) {
throw new Mp4jException("froms array length must be equal to slaveNum");
}
if (tos.length != slaveNum) {
throw new Mp4jException("tos array length must be equal to slaveNum");
}
if (slaveNum == 1) {
return arrData;
}
CommUtils.isfromsTosLegal(froms, tos);
try {
this.operand = operand;
this.operand.setCollective(Collective.ALL_GATHER);
this.operand.setContainer(Container.ARRAY);
ArrayMetaData<T> arrayMetaData = new ArrayMetaData<>();
arrayMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setStep(0)
.setSum(1)
.setCollective(Collective.ALL_GATHER)
.insert(rank, froms[rank], tos[rank]);
arrayMetaData.setArrData(arrData);
return (T)ringAllgather(arrayMetaData, Container.ARRAY).getArrData();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Takes elements from all processes and gathers them to all processes.
* Similar with {@link #allgatherArray(Object, Operand, int[], int[])}, this container is map.
* @param mapData map data
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @return map list
* @throws Mp4jException
*/
public <T> List<Map<String, T>> allgatherMap(Map<String, T> mapData, Operand operand) throws Mp4jException {
if (slaveNum == 1) {
return Arrays.asList(mapData);
}
try {
this.operand = operand;
this.operand.setCollective(Collective.ALL_GATHER);
this.operand.setContainer(Container.MAP);
MapMetaData<T> mapMetaData = new MapMetaData<>();
mapMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setStep(0)
.setSum(1)
.setCollective(Collective.ALL_GATHER)
.insert(rank, mapData.size());
List<Map<String, T>> listMap = new ArrayList<>(slaveNum);
listMap.add(mapData);
mapMetaData.setMapDataList(listMap);
MapMetaData<T> retMapMetaData = (MapMetaData<T>)ringAllgather(mapMetaData, Container.MAP).convertToMapMetaData();
return retMapMetaData.getMapDataList();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
private <T> MetaData<T> ringAllgather(MetaData thisMetaData, Container container) throws Mp4jException {
try {
List<Map<String, T>> retMapList = null;
if (container == Container.MAP){
retMapList = new ArrayList<>(slaveNum);
for (int i = 0; i < slaveNum; i++) {
retMapList.add(Collections.emptyMap());
}
retMapList.set(rank, (Map<String, T>)thisMetaData.getMapDataList().get(0));
}
// send first block
sendQueue.put(thisMetaData);
// recv slaveNum - 1 times, send slaveNum - 2
for (int step = 1; step < slaveNum; step++) {
recvQueue.push(thisMetaData);
MetaData<T> thatMetaData = recvResultQueue.take();
if (container == Container.MAP) {
int realOriginRank = thatMetaData.convertToMapMetaData().getRanks().get(0);
retMapList.set(realOriginRank, thatMetaData.getMapDataList().get(0));
}
thisMetaData = thatMetaData;
thatMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setCollective(Collective.ALL_GATHER);
if (step < slaveNum - 1) {
sendQueue.put(thatMetaData);
}
}
if (container == Container.MAP) {
thisMetaData.setMapDataList(retMapList);
}
barrier();
} catch (Exception e) {
throw new Mp4jException(e);
}
return thisMetaData;
}
/**
* Broadcast array in root process to all other processe(included itself).
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param from start position to broadcast
* @param to end position to broadcast
* @param rootRank root rank
* @return root array, interval is [from, to)
* @throws Mp4jException
*/
public <T> T broadcastArray(T arrData, Operand operand, int from, int to, int rootRank) throws Mp4jException {
if (slaveNum == 1) {
return arrData;
}
CommUtils.isFromToLegal(from, to);
try {
int[] froms = new int[slaveNum];
int[] tos = new int[slaveNum];
int avg = (to - from) / slaveNum;
int fromidx = from;
for (int r = 0; r < slaveNum; r++) {
froms[r] = fromidx;
tos[r] = fromidx + avg;
fromidx += avg;
}
tos[slaveNum - 1] = to;
T scatterArr = scatterArray(arrData, operand, froms, tos, rootRank);
return allgatherArray(scatterArr, operand, froms, tos);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Broadcast single value in root process to all other processes(included itself).
* @param value value to be broadcast(only root process is valid)
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param rootRank root rank
* @return the value of root process.
* @throws Mp4jException
*/
public <T> T broadcast(T value, Operand operand, int rootRank) throws Mp4jException {
if (operand instanceof DoubleOperand) {
double []doubleArr = new double[1];
doubleArr[0] = (Double) value;
doubleArr = broadcastArray(doubleArr, operand, 0, doubleArr.length, rootRank);
return (T)Double.valueOf(doubleArr[0]);
} else if (operand instanceof FloatOperand) {
float []floatArr = new float[1];
floatArr[0] = (Float) value;
floatArr = broadcastArray(floatArr, operand, 0, floatArr.length, rootRank);
return (T)Float.valueOf(floatArr[0]);
} else if (operand instanceof IntOperand) {
int []intArr = new int[1];
intArr[0] = (Integer) value;
intArr = broadcastArray(intArr, operand, 0, intArr.length, rootRank);
return (T)Integer.valueOf(intArr[0]);
} else if (operand instanceof LongOperand) {
long []longArr = new long[1];
longArr[0] = (Long) value;
longArr = broadcastArray(longArr, operand, 0, longArr.length, rootRank);
return (T)Long.valueOf(longArr[0]);
} else if (operand instanceof ObjectOperand) {
T []objectArr = (T[]) Array.newInstance(value.getClass(), 1);
objectArr[0] = value;
objectArr = broadcastArray(objectArr, operand, 0, objectArr.length, rootRank);
return objectArr[0];
} else if (operand instanceof StringOperand) {
String []stringArr = new String[1];
stringArr[0] = (String) value;
stringArr = broadcastArray(stringArr, operand, 0, stringArr.length, rootRank);
return (T)stringArr[0];
} else if (operand instanceof ShortOperand) {
short []shortArr = new short[1];
shortArr[0] = (Short) value;
shortArr = broadcastArray(shortArr, operand, 0, shortArr.length, rootRank);
return (T)Short.valueOf(shortArr[0]);
} else if (operand instanceof ByteOperand) {
byte []byteArr = new byte[1];
byteArr[0] = (Byte) value;
byteArr = broadcastArray(byteArr, operand, 0, byteArr.length, rootRank);
return (T)Byte.valueOf(byteArr[0]);
} else {
throw new Mp4jException("unknown operand:" + operand);
}
}
/**
* Broadcast map in root process to all other processes(included itself),
* similar with {@link #broadcastArray(Object, Operand, int, int, int)}, but this container is map.
* @param mapData map data
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param rootRank root rank
* @return the root map.
* @throws Mp4jException
*/
public <T> Map<String, T> broadcastMap(Map<String, T> mapData, Operand operand, int rootRank) throws Mp4jException {
if (slaveNum == 1) {
return mapData;
}
try {
List<Map<String, T>> listMapData = new ArrayList<>();
if (rank == rootRank) {
listMapData = new ArrayList<>(slaveNum);
for (int i = 0; i < slaveNum; i++) {
listMapData.add(new HashMap<>((int)((mapData.size() / slaveNum) * 1.2)));
}
for (Map.Entry<String, T> entry : mapData.entrySet()) {
String key = entry.getKey();
T val = entry.getValue();
int idx = key.hashCode() % slaveNum;
if (idx < 0) {
idx += slaveNum;
}
listMapData.get(idx).put(key, val);
}
}
Map<String, T> scatteredMap = scatterMap(listMapData, operand, rootRank);
List<Map<String, T>> broadcasedMapList = allgatherMap(scatteredMap, operand);
Map<String, T> retMap = broadcasedMapList.get(0);
for (int i = 1; i < broadcasedMapList.size(); i++) {
Map<String, T> retMapTemp = broadcasedMapList.get(i);
for (Map.Entry<String, T> entry : retMapTemp.entrySet()) {
retMap.put(entry.getKey(), entry.getValue());
}
}
return retMap;
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Send chunks of an array to different processes, the data container is array.
* process i receive data interval [recvfroms[i], recvtos[i]) from root process, placed in the same positions,
* s.t. recvfroms[i] ≤ recvtos[i], recvfroms[i] ≥ recvtos[i-1]
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param recvfroms receiving start positions, included
* @param recvtos receiving end positions, excluded
* @param rootRank root rank
* @return rank i receives data from root process, placed in [recvfroms[i], recvtos[i])
* @throws Mp4jException
*/
public <T> T scatterArray(T arrData, Operand operand, int[] recvfroms, int[] recvtos, int rootRank) throws Mp4jException {
try {
if (recvfroms.length != slaveNum) {
throw new Mp4jException("recvfroms array length must be equal to slaveNum");
}
if (recvtos.length != slaveNum) {
throw new Mp4jException("recvtos array length must be equal to slaveNum");
}
if (slaveNum == 1) {
return arrData;
}
CommUtils.isfromsTosLegal(recvfroms, recvtos);
this.operand = operand;
this.operand.setCollective(Collective.SCATTER);
this.operand.setContainer(Container.ARRAY);
ArrayMetaData<T> arrayMetaData = new ArrayMetaData<>();
arrayMetaData.setSrcRank(rank)
.setDestRank(-1)
.setStep(0)
.setSum(0)
.setCollective(Collective.SCATTER);
for (int i = 0; i < slaveNum; i++) {
arrayMetaData.insert(i, recvfroms[i], recvtos[i]);
}
arrayMetaData.setArrData(arrData);
return (T)binaryTreeScatter(rootRank, arrayMetaData, Container.ARRAY).convertToArrayMetaData().getArrData();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Send chunks of data to different processes, the data container is map.
* rank i receive data mapDataList.get(i) from root process.
* @param mapDataList list of maps to be scattered.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param rootRank root rank
* @return rank i receives data mapDataList.get(i) from root process.
* @throws Mp4jException
*/
public <T> Map<String, T> scatterMap(List<Map<String, T>> mapDataList, Operand operand, int rootRank) throws Mp4jException {
try {
if (rank == rootRank && mapDataList.size() != slaveNum) {
throw new Mp4jException("mapDataList size must be equal to slaveNum");
}
if (slaveNum == 1) {
return mapDataList.get(0);
}
LOG.info("entry scater map...");
this.operand = operand;
this.operand.setCollective(Collective.SCATTER);
this.operand.setContainer(Container.MAP);
MapMetaData<T> MapMetaData = new MapMetaData<>();
MapMetaData.setSrcRank(rank)
.setDestRank(-1)
.setStep(0)
.setSum(0)
.setCollective(Collective.SCATTER);
if (rank == rootRank) {
for (int i = 0; i < slaveNum; i++) {
MapMetaData.insert(i, mapDataList.get(i).size());
}
MapMetaData.setMapDataList(mapDataList);
}
LOG.info("will call binary tree scatter...");
MapMetaData<T> metaData = (MapMetaData<T>)binaryTreeScatter(rootRank, MapMetaData, Container.MAP).convertToMapMetaData();
Map<String, T> retMap = null;
List<Map<String, T>> listMapData = metaData.getMapDataList();
for (int i = 0; i < metaData.getSegNum(); i++) {
int curRank = metaData.getRank(i);
if (curRank == rank) {
retMap = listMapData.get(i);
}
}
listMapData.clear();
if (retMap == null) {
throw new Mp4jException("scatter error retmap must not be null!");
}
return retMap;
} catch (Exception e) {
throw new Mp4jException(e);
}
}
protected <T> List<Map<String, T>> scatterMapSpecial(List<List<Map<String, T>>> mapDataListList, Operand operand, int rootRank) throws Mp4jException {
try {
if (rank == rootRank && mapDataListList.size() != slaveNum) {
throw new Mp4jException("mapDataList size must be equal to slaveNum");
}
if (slaveNum == 1) {
return mapDataListList.get(0);
}
LOG.info("entry special scatter map...");
this.operand = operand;
this.operand.setCollective(Collective.SCATTER);
this.operand.setContainer(Container.MAP);
MapMetaData<T> mapMetaData = new MapMetaData<>();
mapMetaData.setSrcRank(rank)
.setDestRank(-1)
.setStep(0)
.setSum(0)
.setCollective(Collective.SCATTER);
if (rank == rootRank) {
List<Map<String, T>> mapDataListForScatter = new ArrayList<>();
for (int i = 0; i < slaveNum; i++) {
List<Map<String, T>> mapDataList = mapDataListList.get(i);
for (int j = 0; j < mapDataList.size(); j++) {
mapDataListForScatter.add(mapDataList.get(j));
mapMetaData.insert(i, mapDataList.get(j).size());
}
}
LOG.info("special root map data list size:" + mapDataListForScatter.size() + ", metadata:" + mapMetaData);
mapMetaData.setMapDataList(mapDataListForScatter);
}
LOG.info("will call binary tree scatter...");
MapMetaData<T> metaData = (MapMetaData<T>)binaryTreeScatter(rootRank, mapMetaData, Container.MAP).convertToMapMetaData();
LOG.info("binary tree scatter finished!");
List<Map<String, T>> retMapList = new ArrayList<>();
List<Map<String, T>> listMapData = metaData.getMapDataList();
for (int i = 0; i < metaData.getSegNum(); i++) {
int curRank = metaData.getRank(i);
if (curRank == rank) {
retMapList.add(listMapData.get(i));
}
}
listMapData.clear();
return retMapList;
} catch (Exception e) {
throw new Mp4jException(e);
}
}
private <T> MetaData<T> scatterSend(MetaData metaData,
List<Integer> sendInfo,
Container container) throws Mp4jException {
MetaData newMetaData = MetaData.newMetaData(container);
int newSrcRank = sendInfo.get(0);
int newDestRank = sendInfo.get(1);
int newRankFrom = sendInfo.get(2);
int newRankTo = sendInfo.get(3);
newMetaData.setSrcRank(newSrcRank);
newMetaData.setDestRank(newDestRank);
newMetaData.setCollective(Collective.SCATTER);
int segNum = metaData.getSegNum();
if (container == Container.ARRAY) {
List<Integer> ranks = metaData.convertToArrayMetaData().getRanks();
List<Integer> froms = metaData.convertToArrayMetaData().getSegFroms();
List<Integer> tos = metaData.convertToArrayMetaData().getSegTos();
for (int i = 0; i < segNum; i++) {
int rank = ranks.get(i);
int from = froms.get(i);
int to = tos.get(i);
if (rank >= newRankFrom && rank <= newRankTo) {
newMetaData.insert(rank, from, to);
}
}
newMetaData.setArrData(metaData.getArrData());
} else if (container == Container.MAP) {
List<Integer> dataNums = metaData.convertToMapMetaData().getDataNums();
List<Integer> ranks = metaData.convertToMapMetaData().getRanks();
List<Map<String, T>> mapList = metaData.convertToMapMetaData().getMapDataList();
List<Map<String, T>> newMapList = new ArrayList<>();
for (int i = 0; i < segNum; i++) {
int rank = ranks.get(i);
int dataNum = dataNums.get(i);
if (rank >= newRankFrom && rank <= newRankTo) {
newMetaData.insert(rank, dataNum);
newMapList.add(mapList.get(i));
}
}
newMetaData.setMapDataList(newMapList);
} else {
throw new Mp4jException("unsupport MetaData type in scatter send");
}
return newMetaData;
}
private <T> MetaData<T> binaryTreeScatter(int rootRank, MetaData thisMetaData, Container container) throws Mp4jException {
try {
LOG.info("entry binary tree scatter ...");
Map<Integer, List<List<Integer>>> allocateMap = ScatterAllocate.allocate(slaveNum, rootRank);
Map<Integer, Integer> recvNumMap = ScatterAllocate.recvNum(allocateMap);
// get this rank's send task list
List<List<Integer>> thisSendTaskList = allocateMap.getOrDefault(rank, Collections.emptyList());
// get recv number
Integer recvNum = recvNumMap.getOrDefault(rank, 0);
if (recvNum >= 2) {
throw new Mp4jException("scatter error, recv num must <= 1");
}
LOG.info("this send task list:" + thisSendTaskList);
LOG.info("this recv num:" + recvNum);
// every process only recv once data except the root process
int thisSendCursor = 0;
if (rank == rootRank) {
if (recvNum > 0) {
List<Integer> half0 = thisSendTaskList.get(thisSendCursor++);
sendQueue.put(scatterSend(thisMetaData, half0, container));
if (thisSendCursor < thisSendTaskList.size()) {
List<Integer> half1 = thisSendTaskList.get(thisSendCursor);
int toRank = half1.get(2);
if (toRank == 0 || toRank == slaveNum / 2) {
sendQueue.put(scatterSend(thisMetaData, half1, container));
thisSendCursor ++;
}
}
} else {
for (; thisSendCursor < thisSendTaskList.size(); thisSendCursor++) {
sendQueue.put(scatterSend(thisMetaData, thisSendTaskList.get(thisSendCursor), container));
}
}
}
// recv at most once & send
if (recvNum > 0) {
recvQueue.push(thisMetaData);
MetaData<T> thatMetaData = recvResultQueue.take();
thisMetaData = thatMetaData;
thisMetaData.setCollective(Collective.SCATTER);
for (int cursor = thisSendCursor; cursor < thisSendTaskList.size(); cursor++) {
sendQueue.put(scatterSend(thisMetaData, thisSendTaskList.get(cursor), container));
}
}
barrier();
} catch (Exception e) {
throw new Mp4jException(e);
}
return thisMetaData;
}
/**
* ReduceScatterArray operation can ve viewed as a combination of {@link #reduceArray(Object, Operand, IOperator, int, int, int)}
* and {@link #scatterArray(Object, Operand, int[], int[], int)}.
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param from start position of arrData to be ReduceScattered
* @param counts rank i receive number of counts[i] elements reducescatter result.
* @return rank 0 receive interval is [from, counts[0]),
* rank 1 receive interval is [from+count[0], from+count[0]+count[1]), ...
* @throws Mp4jException
*/
public <T> T reduceScatterArray(T arrData, Operand operand, IOperator operator, int from, int[] counts) throws Mp4jException {
if (counts.length != slaveNum) {
throw new Mp4jException("counts array length must be equal to slaveNum");
}
if (slaveNum == 1) {
return arrData;
}
CommUtils.isFromCountsLegal(from, counts);
try {
int []froms = CommUtils.getFromsFromCount(from, counts, slaveNum);
int []tos = CommUtils.getTosFromCount(from, counts, slaveNum);
this.operand = operand;
this.operand.setCollective(Collective.REDUCE_SCATTER);
this.operand.setContainer(Container.ARRAY);
this.operand.setOperator(operator);
int sendRank = (rank - 1 + slaveNum) % slaveNum;
ArrayMetaData<T> arrayMetaData = new ArrayMetaData<>();
arrayMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setStep(0)
.setSum(1)
.setCollective(Collective.REDUCE_SCATTER)
.insert(rank, froms[sendRank], tos[sendRank]);
arrayMetaData.setArrData(arrData);
return (T)ringReduceScatter(arrayMetaData, Container.ARRAY).getArrData();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
protected <T> List<Map<String, T>> reduceScatterMapSpecial(List<List<Map<String, T>>> mapDataListList, Operand operand, IOperator operator) throws Mp4jException {
if (slaveNum == 1) {
return mapDataListList.get(0);
}
if (mapDataListList.size() != slaveNum) {
throw new Mp4jException("mapDataListList size=" + mapDataListList.size() + ", must be equal to slaveNum=" + slaveNum);
}
int blockNum = mapDataListList.get(0).size();
try {
this.operand = operand;
this.operand.setCollective(Collective.REDUCE_SCATTER);
this.operand.setContainer(Container.MAP);
this.operand.setOperator(operator);
MapMetaData<T> mapMetaData = new MapMetaData<>();
mapMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setStep(0)
.setSum(1)
.setCollective(Collective.REDUCE_SCATTER);
List<Map<String, T>> allmapDataList = new ArrayList<>();
for (int i = 0; i < mapDataListList.size(); i++) {
List<Map<String, T>> maps = mapDataListList.get(i);
for (int j = 0; j < maps.size(); j++) {
allmapDataList.add(maps.get(j));
}
}
List<Map<String, T>> sendMapList = new ArrayList<>();
int sendIdx = (rank - 1 + slaveNum) % slaveNum;
for (int t = 0; t < blockNum; t++) {
int idx = sendIdx * blockNum + t;
mapMetaData.insert(rank, allmapDataList.get(idx).size());
sendMapList.add(allmapDataList.get(idx));
}
mapMetaData.setMapDataList(sendMapList);
// send first block
sendQueue.put(mapMetaData);
// recv slaveNum - 1 times, send slaveNum - 2
for (int step = 1; step < slaveNum; step++) {
MapMetaData recvMapMetaData = new MapMetaData();
List<Map<String, T>> recvMapList = new ArrayList<>();
int recvIdx = (rank - (step + 1) + slaveNum) % slaveNum;
for (int t = 0; t < blockNum; t++) {
int idx = recvIdx * blockNum + t;
recvMapMetaData.insert(rank, allmapDataList.get(idx).size());
recvMapList.add(allmapDataList.get(idx));
}
recvMapMetaData.setMapDataList(recvMapList);
recvQueue.push(recvMapMetaData);
MetaData<T> thatMetaData = recvResultQueue.take();
mapMetaData = thatMetaData.convertToMapMetaData();
mapMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setCollective(Collective.REDUCE_SCATTER);
if (step < slaveNum - 1) {
sendQueue.put(mapMetaData);
}
}
barrier();
return mapMetaData.getMapDataList();
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* reduceScatterMap operation can be viewed as a combination of {@link #reduceMap(Map, Operand, IOperator, int)}
* and {@link #scatterMap(List, Operand, int)}.
* @param mapDataList list of map
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @return rank i process will receive reduce of mapDataList.get(i) in all process
* @throws Mp4jException
*/
public <T> Map<String, T> reduceScatterMap(List<Map<String, T>> mapDataList, Operand operand, IOperator operator) throws Mp4jException {
if (slaveNum == 1) {
return mapDataList.get(0);
}
if (mapDataList.size() != slaveNum) {
throw new Mp4jException("mapDataList size=" + mapDataList.size() + ", must be equal to slaveNum=" + slaveNum);
}
try {
this.operand = operand;
this.operand.setCollective(Collective.REDUCE_SCATTER);
this.operand.setContainer(Container.MAP);
this.operand.setOperator(operator);
MapMetaData<T> mapMetaData = new MapMetaData<>();
mapMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setStep(0)
.setSum(1)
.setCollective(Collective.REDUCE_SCATTER);
mapMetaData.setMapDataList(mapDataList);
MapMetaData<T> retMapMetaData = (MapMetaData<T>)ringReduceScatter(mapMetaData, Container.MAP).convertToMapMetaData();
return retMapMetaData.getMapDataList().get(0);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
private <T> MetaData<T> ringReduceScatter(MetaData thisMetaData, Container container) throws Mp4jException {
try {
List<Map<String, T>> storeMapList = null;
if (container == Container.MAP){
storeMapList = thisMetaData.getMapDataList();
int sendIdx = (rank - 1 + slaveNum) % slaveNum;
thisMetaData.insert(rank, storeMapList.get(sendIdx).size());
thisMetaData.setMapDataList(Arrays.asList(storeMapList.get(sendIdx)));
}
// send first block
sendQueue.put(thisMetaData);
// recv slaveNum - 1 times, send slaveNum - 2
for (int step = 1; step < slaveNum; step++) {
if (container == Container.ARRAY) {
recvQueue.push(thisMetaData);
} else if (container == Container.MAP) {
MapMetaData recvMapMetaData = new MapMetaData();
int recvIdx = (rank - (step + 1) + slaveNum) % slaveNum;
recvMapMetaData.insert(rank, storeMapList.get(recvIdx).size());
recvMapMetaData.setMapDataList(Arrays.asList(storeMapList.get(recvIdx)));
recvQueue.push(recvMapMetaData);
}
MetaData<T> thatMetaData = recvResultQueue.take();
thisMetaData = thatMetaData;
thatMetaData.setSrcRank(rank)
.setDestRank((rank + 1) % slaveNum)
.setCollective(Collective.REDUCE_SCATTER);
if (step < slaveNum - 1) {
sendQueue.put(thatMetaData);
}
}
barrier();
} catch (Exception e) {
throw new Mp4jException(e);
}
return thisMetaData;
}
/**
* reduce all array elements, reduced result locate in root process
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param from start position to reduce
* @param to end position to reduce
* @param rootRank root rank
* @return if this process is root, reduced array is returned, otherwise,
* is invalid array, contains intermediate result.
* @throws Mp4jException
*/
public <T> T reduceArray(T arrData, Operand operand, IOperator operator, int from, int to, int rootRank) throws Mp4jException {
if (slaveNum == 1) {
return arrData;
}
CommUtils.isFromToLegal(from, to);
try {
int[] counts = new int[slaveNum];
int avg = (to - from) / slaveNum;
for (int r = 0; r < slaveNum; r++) {
counts[r] = avg;
}
counts[slaveNum - 1] = (to - from) - ((slaveNum - 1) * avg);
int[] froms = new int[slaveNum];
int[] tos = new int[slaveNum];
int fromidx = from;
for (int r = 0; r < slaveNum; r++) {
froms[r] = fromidx;
tos[r] = fromidx + avg;
fromidx += avg;
}
tos[slaveNum - 1] = to;
T reduceScatteredArr = reduceScatterArray(arrData, operand, operator, from, counts);
return gatherArray(reduceScatteredArr, operand, froms, tos, rootRank);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* single value reduce
* @param value value to be reduced
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param rootRank root rank
* @return if this process is root, reduced value is returned, otherwise, is a invalid value.
* @throws Mp4jException
*/
public <T> T reduce(T value, Operand operand, IOperator operator, int rootRank) throws Mp4jException {
if (operand instanceof DoubleOperand) {
double []doubleArr = new double[1];
doubleArr[0] = (Double) value;
doubleArr = reduceArray(doubleArr, operand, operator, 0, doubleArr.length, rootRank);
return (T)Double.valueOf(doubleArr[0]);
} else if (operand instanceof FloatOperand) {
float []floatArr = new float[1];
floatArr[0] = (Float) value;
floatArr = reduceArray(floatArr, operand, operator, 0, floatArr.length, rootRank);
return (T)Float.valueOf(floatArr[0]);
} else if (operand instanceof IntOperand) {
int []intArr = new int[1];
intArr[0] = (Integer) value;
intArr = reduceArray(intArr, operand, operator, 0, intArr.length, rootRank);
return (T)Integer.valueOf(intArr[0]);
} else if (operand instanceof LongOperand) {
long []longArr = new long[1];
longArr[0] = (Long) value;
longArr = reduceArray(longArr, operand, operator, 0, longArr.length, rootRank);
return (T)Long.valueOf(longArr[0]);
} else if (operand instanceof ObjectOperand) {
T []objectArr = (T[]) Array.newInstance(value.getClass(), 1);
objectArr[0] = value;
objectArr = reduceArray(objectArr, operand, operator, 0, objectArr.length, rootRank);
return objectArr[0];
} else if (operand instanceof StringOperand) {
String []stringArr = new String[1];
stringArr[0] = (String) value;
stringArr = reduceArray(stringArr, operand, operator, 0, stringArr.length, rootRank);
return (T)stringArr[0];
} else if (operand instanceof ShortOperand) {
short []shortArr = new short[1];
shortArr[0] = (Short) value;
shortArr = reduceArray(shortArr, operand, operator, 0, shortArr.length, rootRank);
return (T)Short.valueOf(shortArr[0]);
} else if (operand instanceof ByteOperand) {
byte []byteArr = new byte[1];
byteArr[0] = (Byte) value;
byteArr = reduceArray(byteArr, operand, operator, 0, byteArr.length, rootRank);
return (T)Byte.valueOf(byteArr[0]);
} else {
throw new Mp4jException("unknown operand:" + operand);
}
}
/**
* Similar with {@link #reduceArray(Object, Operand, IOperator, int, int, int)},
* but the container is map.
* @param mapData map data
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param rootRank root rank
* @return if this process is root, reduced map is returned, otherwise, invalid map or null is returned.
* @throws Mp4jException
*/
public <T> Map<String, T> reduceMap(Map<String, T> mapData, Operand operand, IOperator operator, int rootRank) throws Mp4jException {
if (slaveNum == 1) {
return mapData;
}
try {
List<Map<String, T>> listMapData = new ArrayList<>(slaveNum);
for (int i = 0; i < slaveNum; i++) {
listMapData.add(new HashMap<>(Math.max((int)((mapData.size() / slaveNum) * 1.2), 1)));
}
for (Map.Entry<String, T> entry : mapData.entrySet()) {
String key = entry.getKey();
T val = entry.getValue();
int idx = key.hashCode() % slaveNum;
if (idx < 0) {
LOG.info(key + ", code:" + idx);
idx += slaveNum;
}
listMapData.get(idx).put(key, val);
}
Map<String, T> reduceScatteredMap = reduceScatterMap(listMapData, operand, operator);
return gatherMap(reduceScatteredMap, operand, rootRank);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
protected static class Mp4jSetSerializer<T> extends Serializer<Set<T>> {
Serializer<T> valSerializer;
Class<T> valType;
public Mp4jSetSerializer(Serializer<T> valSerializer, Class<T> valType) {
this.valSerializer = valSerializer;
this.valType = valType;
}
@Override
public void write(Kryo kryo, Output output, Set<T> object) {
output.writeInt(object.size());
for (T val : object) {
valSerializer.write(kryo, output, val);
}
}
@Override
public Set<T> read(Kryo kryo, Input input, Class<Set<T>> type) {
int size = input.readInt();
Set<T> set = new HashSet<>(size);
for (int i = 0; i < size; i++) {
set.add(valSerializer.read(kryo, input, this.valType));
}
return set;
}
}
protected static class Mp4jListSerializer<T> extends Serializer<List<T>> {
Serializer<T> valSerializer;
Class<T> valType;
public Mp4jListSerializer(Serializer<T> valSerializer, Class<T> valType) {
this.valSerializer = valSerializer;
this.valType = valType;
}
@Override
public void write(Kryo kryo, Output output, List<T> object) {
output.writeInt(object.size());
for (T val : object) {
valSerializer.write(kryo, output, val);
}
}
@Override
public List<T> read(Kryo kryo, Input input, Class<List<T>> type) {
int size = input.readInt();
List<T> list = new ArrayList<T>(size);
for (int i = 0; i < size; i++) {
list.add(valSerializer.read(kryo, input, this.valType));
}
return list;
}
}
/**
* Set union, the set with the same key will be reduced(union) together in the root process.
* @param mapData map set data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element obejct class
* @return if this process is root, the set with the same key will be reduced together,
* otherwise, invalid map or null is returned.
* @throws Mp4jException
*/
public <T> Map<String, Set<T>> reduceMapSetUnion(Map<String, Set<T>> mapData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jSetSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<Set<T>>() {
@Override
public Set<T> apply(Set<T> o1, Set<T> o2) {
for (T val : o2) {
o1.add(val);
}
return o1;
}
};
return reduceMap(mapData, operand, operator, rootRank);
}
/**
* Set union
* @param setData set data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return if this process is root, set unison is returned,
* otherwise invalid set or null is returned.
* @throws Mp4jException
*/
public <T> Set<T> reduceSetUnion(Set<T> setData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, Set<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", setData);
Map<String, Set<T>> mapReturn = reduceMapSetUnion(mapTemp, rootRank, elementSerializer, elementType);
if (mapReturn != null) {
return mapReturn.get("key");
} else {
return null;
}
}
/**
* Set intersection, the set with the same key will be reduced(intersect) together.
* @param mapData map set data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return if this process is root, the set with the same key will be reduced(intersect) together.
* otherwise, invalid map is returned.
* @throws Mp4jException
*/
public <T> Map<String, Set<T>> reduceMapSetIntersection(Map<String, Set<T>> mapData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jSetSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<Set<T>>() {
@Override
public Set<T> apply(Set<T> o1, Set<T> o2) {
o1.retainAll(o2);
return o1;
}
};
return reduceMap(mapData, operand, operator, rootRank);
}
/**
* Set intersection.
* @param setData set data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return if this process is root, intersection is returned,
* otherwise, invalid set or null is returned.
* @throws Mp4jException
*/
public <T> Set<T> reduceSetIntersection(Set<T> setData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, Set<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", setData);
Map<String, Set<T>> mapReturn = reduceMapSetIntersection(mapTemp, rootRank, elementSerializer, elementType);
if (mapReturn != null) {
return mapReturn.get("key");
} else {
return null;
}
}
/**
* List concat, the lists with the same key will be reduced(concat) together.
* @param mapData map list data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return if this process is root, the lists with the same key will be reduced(concat) together,
* otherwise, invalid map or null is returned.
* @throws Mp4jException
*/
public <T> Map<String, List<T>> reduceMapListConcat(Map<String, List<T>> mapData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jListSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<List<T>>() {
@Override
public List<T> apply(List<T> o1, List<T> o2) {
for (T val : o2) {
o1.add(val);
}
return o1;
}
};
return reduceMap(mapData, operand, operator, rootRank);
}
/**
* List concat.
* @param listData list data
* @param rootRank root rank
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return if this process is root, the concated list is returned,
* otherwise, invalid list or null is returned.
* @throws Mp4jException
*/
public <T> List<T> reduceListConcat(List<T> listData, int rootRank, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, List<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", listData);
Map<String, List<T>> mapReturn = reduceMapListConcat(mapTemp, rootRank, elementSerializer, elementType);
if (mapReturn != null) {
return mapReturn.get("key");
} else {
return null;
}
}
/**
* Different with reduce operation which only root process contains final reduced result,
* while all process receive the same reduced result in allreduce operation.
*
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param from start position to reduce
* @param to end position to reduce
* @return arrData[i](rank) = reduce(arrData[i](rank_0), arrData[i](rank_1), ..., arrData[i](rank_slavenumber-1)),
* @throws Mp4jException
*/
public <T> T allreduceArray(T arrData, Operand operand, IOperator operator, int from, int to) throws Mp4jException {
if (slaveNum == 1) {
return arrData;
}
CommUtils.isFromToLegal(from, to);
try {
int[] counts = new int[slaveNum];
int avg = (to - from) / slaveNum;
for (int r = 0; r < slaveNum; r++) {
counts[r] = avg;
}
counts[slaveNum - 1] = (to - from) - ((slaveNum - 1) * avg);
int[] froms = new int[slaveNum];
int[] tos = new int[slaveNum];
int fromidx = from;
for (int r = 0; r < slaveNum; r++) {
froms[r] = fromidx;
tos[r] = fromidx + avg;
fromidx += avg;
}
tos[slaveNum - 1] = to;
T reduceScatteredArr = reduceScatterArray(arrData, operand, operator, from, counts);
return allgatherArray(reduceScatteredArr, operand, froms, tos);
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Similar with {@link #allreduceArray(Object, Operand, IOperator, int, int)},
* but it's realized by rpc communication. It is suited to small data.
*
* @param arrData data array, each process have the same length.
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param <T>
* @return arrData[i](rank) = reduce(arrData[i](rank_0), arrData[i](rank_1), ..., arrData[i](rank_slavenumber-1)),
* @throws Mp4jException
*/
public <T> T allreduceArrayRpc(T arrData, Operand operand, IOperator operator) throws Mp4jException {
if (slaveNum == 1) {
return arrData;
}
if (operand instanceof DoubleOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
double []retArray = (double[])ret.get();
double []thisArray = (double[])arrData;
IDoubleOperator doubleOperator = (IDoubleOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = doubleOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else if (operand instanceof FloatOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
float []retArray = (float[])ret.get();
float []thisArray = (float[])arrData;
IFloatOperator floatOperator = (IFloatOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = floatOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else if (operand instanceof IntOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
int []retArray = (int[])ret.get();
int []thisArray = (int[])arrData;
IIntOperator intOperator = (IIntOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = intOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else if (operand instanceof LongOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
long []retArray = (long[])ret.get();
long []thisArray = (long[])arrData;
ILongOperator longOperator = (ILongOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = longOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else if (operand instanceof ObjectOperand) {
ObjectOperand objectOperand = (ObjectOperand)operand;
objectOperand.setOperator(operator);
ArrayPrimitiveWritable ret = server.arrayAllReduce(new ArrayPrimitiveWritable(objectOperand.convertToBytes(arrData)), rank);
Input input = new Input(new ByteArrayInputStream((byte[])ret.get()));
T retx = (T)objectOperand.readFromBytes(arrData, input, slaveNum);
input.close();
return retx;
} else if (operand instanceof StringOperand) {
String[] thisArray = (String []) arrData;
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(1024);
Output output = new Output(byteArrayOutputStream);
KryoUtils.getKryo().writeClassAndObject(output, thisArray);
output.close();
ArrayPrimitiveWritable ret = server.arrayAllReduce(new ArrayPrimitiveWritable(byteArrayOutputStream.toByteArray()), rank);
Input input = new Input(new ByteArrayInputStream((byte[])ret.get()));
String []retArray;
IStringOperator stringOperator = (IStringOperator) operator;
for (int i = 0; i < slaveNum; i++) {
retArray = (String[])KryoUtils.getKryo().readClassAndObject(input);
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = stringOperator.apply(thisArray[j], retArray[j]);
}
}
}
input.close();
} else if (operand instanceof ShortOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
short []retArray = (short[])ret.get();
short []thisArray = (short[])arrData;
IShortOperator shortOperator = (IShortOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = shortOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else if (operand instanceof ByteOperand) {
ArrayPrimitiveWritable ret = server.primitiveArrayAllReduce(new ArrayPrimitiveWritable(arrData), rank);
byte []retArray = (byte[])ret.get();
byte []thisArray = (byte[])arrData;
IByteOperator byteOperator = (IByteOperator) operator;
int idx = 0;
for (int i = 0; i < slaveNum; i++) {
if (i == 0) {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = retArray[idx + j];
}
} else {
for (int j = 0; j < thisArray.length; j++) {
thisArray[j] = byteOperator.apply(thisArray[j], retArray[idx + j]);
}
}
idx += thisArray.length;
}
} else {
throw new Mp4jException("unknown operand:" + operand);
}
return arrData;
}
/**
* Similar with {@link #allreduceArray(Object, Operand, IOperator, int, int)},
* this function only allreduce just only one elements.
* @param value the value for reduce
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @return all processes receive the same reduced value.
* @throws Mp4jException
*/
public <T> T allreduce(T value, Operand operand, IOperator operator) throws Mp4jException {
if (operand instanceof DoubleOperand) {
double []doubleArr = new double[1];
doubleArr[0] = (Double) value;
doubleArr = allreduceArray(doubleArr, operand, operator, 0, doubleArr.length);
return (T)Double.valueOf(doubleArr[0]);
} else if (operand instanceof FloatOperand) {
float []floatArr = new float[1];
floatArr[0] = (Float) value;
floatArr = allreduceArray(floatArr, operand, operator, 0, floatArr.length);
return (T)Float.valueOf(floatArr[0]);
} else if (operand instanceof IntOperand) {
int []intArr = new int[1];
intArr[0] = (Integer) value;
intArr = allreduceArray(intArr, operand, operator, 0, intArr.length);
return (T)Integer.valueOf(intArr[0]);
} else if (operand instanceof LongOperand) {
long []longArr = new long[1];
longArr[0] = (Long) value;
longArr = allreduceArray(longArr, operand, operator, 0, longArr.length);
return (T)Long.valueOf(longArr[0]);
} else if (operand instanceof ObjectOperand) {
T []objectArr = (T[]) Array.newInstance(value.getClass(), 1);
objectArr[0] = value;
objectArr = allreduceArray(objectArr, operand, operator, 0, objectArr.length);
return objectArr[0];
} else if (operand instanceof StringOperand) {
String []stringArr = new String[1];
stringArr[0] = (String) value;
stringArr = allreduceArray(stringArr, operand, operator, 0, stringArr.length);
return (T)stringArr[0];
} else if (operand instanceof ShortOperand) {
short []shortArr = new short[1];
shortArr[0] = (Short) value;
shortArr = allreduceArray(shortArr, operand, operator, 0, shortArr.length);
return (T)Short.valueOf(shortArr[0]);
} else if (operand instanceof ByteOperand) {
byte []byteArr = new byte[1];
byteArr[0] = (Byte) value;
byteArr = allreduceArray(byteArr, operand, operator, 0, byteArr.length);
return (T)Byte.valueOf(byteArr[0]);
} else {
throw new Mp4jException("unknown operand:" + operand);
}
}
/**
* Similar with {@link #allreduce(Object, Operand, IOperator)},
* but it's realized by rpc communication. It is suited to small data.
*
* @param value the value for reduce
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @param <T>
* @return
* @throws Mp4jException
*/
public <T> T allreduceRpc(T value, Operand operand, IOperator operator) throws Mp4jException {
if (operand instanceof DoubleOperand) {
double []doubleArr = new double[1];
doubleArr[0] = (Double) value;
doubleArr = allreduceArrayRpc(doubleArr, operand, operator);
return (T)Double.valueOf(doubleArr[0]);
} else if (operand instanceof FloatOperand) {
float []floatArr = new float[1];
floatArr[0] = (Float) value;
floatArr = allreduceArrayRpc(floatArr, operand, operator);
return (T)Float.valueOf(floatArr[0]);
} else if (operand instanceof IntOperand) {
int []intArr = new int[1];
intArr[0] = (Integer) value;
intArr = allreduceArrayRpc(intArr, operand, operator);
return (T)Integer.valueOf(intArr[0]);
} else if (operand instanceof LongOperand) {
long []longArr = new long[1];
longArr[0] = (Long) value;
longArr = allreduceArrayRpc(longArr, operand, operator);
return (T)Long.valueOf(longArr[0]);
} else if (operand instanceof ObjectOperand) {
T []objectArr = (T[]) Array.newInstance(value.getClass(), 1);
objectArr[0] = value;
objectArr = allreduceArrayRpc(objectArr, operand, operator);
return objectArr[0];
} else if (operand instanceof StringOperand) {
String []stringArr = new String[1];
stringArr[0] = (String) value;
stringArr = allreduceArrayRpc(stringArr, operand, operator);
return (T)stringArr[0];
} else if (operand instanceof ShortOperand) {
short []shortArr = new short[1];
shortArr[0] = (Short) value;
shortArr = allreduceArrayRpc(shortArr, operand, operator);
return (T)Short.valueOf(shortArr[0]);
} else if (operand instanceof ByteOperand) {
byte []byteArr = new byte[1];
byteArr[0] = (Byte) value;
byteArr = allreduceArrayRpc(byteArr, operand, operator);
return (T)Byte.valueOf(byteArr[0]);
} else {
throw new Mp4jException("unknown operand:" + operand);
}
}
/**
* Similar with {@link #allreduceArray(Object, Operand, IOperator, int, int)},
* the container of this function is map, the values which have same key will be reduced,
* different processes can contain the same and different keys.
* @param mapData map to be reduced
* @param operand operand(there are 8 operands in {@link com.fenbi.mp4j.operand.Operands})
* @param operator operator(different operands provide different operator in {@link com.fenbi.mp4j.operator.Operators})
* @return key value map, the values which have same key will be reduced together.
* @throws Mp4jException
*/
public <T> Map<String, T> allreduceMap(Map<String, T> mapData, Operand operand, IOperator operator) throws Mp4jException {
if (slaveNum == 1) {
return mapData;
}
try {
List<Map<String, T>> listMapData = new ArrayList<>(slaveNum);
for (int i = 0; i < slaveNum; i++) {
listMapData.add(new HashMap<>(Math.max((int)((mapData.size() / slaveNum) * 1.2), 1)));
}
for (Map.Entry<String, T> entry : mapData.entrySet()) {
String key = entry.getKey();
T val = entry.getValue();
int idx = key.hashCode() % slaveNum;
if (idx < 0) {
LOG.info(key + ", code:" + idx);
idx += slaveNum;
}
listMapData.get(idx).put(key, val);
}
Map<String, T> reduceScatteredMap = reduceScatterMap(listMapData, operand, operator);
List<Map<String, T>> allreducedMapList = allgatherMap(reduceScatteredMap, operand);
Map<String, T> retMap = allreducedMapList.get(0);
for (int i = 1; i < allreducedMapList.size(); i++) {
Map<String, T> retMapTemp = allreducedMapList.get(i);
for (Map.Entry<String, T> entry : retMapTemp.entrySet()) {
retMap.put(entry.getKey(), entry.getValue());
}
}
return retMap;
} catch (Exception e) {
throw new Mp4jException(e);
}
}
/**
* Set union, the set with the same key will be reduced(union) together.
* @param mapData map set data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return the set with the same key will be reduced together.
* @throws Mp4jException
*/
public <T> Map<String, Set<T>> allreduceMapSetUnion(Map<String, Set<T>> mapData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jSetSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<Set<T>>() {
@Override
public Set<T> apply(Set<T> o1, Set<T> o2) {
for (T val : o2) {
o1.add(val);
}
return o1;
}
};
return allreduceMap(mapData, operand, operator);
}
/**
* Set union
* @param setData set data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return set union result
* @throws Mp4jException
*/
public <T> Set<T> allreduceSetUnion(Set<T> setData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, Set<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", setData);
return allreduceMapSetUnion(mapTemp, elementSerializer, elementType).get("key");
}
/**
* Set intersection, the set with the same key will be reduced(intersect) together.
* @param mapData map set data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return the set with the same key will be reduced(intersect) together.
* @throws Mp4jException
*/
public <T> Map<String, Set<T>> allreduceMapSetIntersection(Map<String, Set<T>> mapData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jSetSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<Set<T>>() {
@Override
public Set<T> apply(Set<T> o1, Set<T> o2) {
o1.retainAll(o2);
return o1;
}
};
return allreduceMap(mapData, operand, operator);
}
/**
* Set intersection
* @param setData set data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return set intersected result
* @throws Mp4jException
*/
public <T> Set<T> allreduceSetIntersection(Set<T> setData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, Set<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", setData);
return allreduceMapSetIntersection(mapTemp, elementSerializer, elementType).get("key");
}
/**
* List concat, the lists with the same key will be reduced(concat) together.
* @param mapData map list data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return the lists with the same key will be reduced(concat) together.
* @throws Mp4jException
*/
public <T> Map<String, List<T>> allreduceMapListConcat(Map<String, List<T>> mapData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Operand operand = Operands.OBJECT_OPERAND(new Mp4jListSerializer<>(elementSerializer, elementType), elementType);
IOperator operator = new IObjectOperator<List<T>>() {
@Override
public List<T> apply(List<T> o1, List<T> o2) {
for (T val : o2) {
o1.add(val);
}
return o1;
}
};
return allreduceMap(mapData, operand, operator);
}
/**
* list concat
* @param listData list data
* @param elementSerializer element object Kryo serializer
* @param elementType element object class
* @return list concated result
* @throws Mp4jException
*/
public <T> List<T> allreduceListConcat(List<T> listData, Serializer<T> elementSerializer, Class<T> elementType) throws Mp4jException {
Map<String, List<T>> mapTemp = new HashMap<>(1);
mapTemp.put("key", listData);
return allreduceMapListConcat(mapTemp, elementSerializer, elementType).get("key");
}
private Input getInput() throws Mp4jException {
Socket recvDataSock = null;
Input input = null;
try {
recvDataSock = getRecvDataSocket();
input = new Input(recvDataSock.getInputStream());
LOG.debug("getInput:" + recvDataSock.isClosed());
} catch (Exception e) {
throw new Mp4jException(e);
}
return input;
}
private Output getOutput(int targetRank) throws Mp4jException {
Socket sendDataSock = null;
Output output = null;
try {
sendDataSock = getSendDataSocket(targetRank);
output = new Output(sendDataSock.getOutputStream());
LOG.debug("getOutput:" + sendDataSock.isClosed());
} catch (Exception e) {
throw new Mp4jException(e);
}
return output;
}
}
