/*
* Copyright 2013 University of Southern California
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.package edu.usc.goffish.gopher.sample;
*/
package edu.usc.pgroup.louvain.hadoop;
import com.sun.org.apache.xerces.internal.xni.grammars.Grammar;
import org.apache.commons.math.optimization.VectorialConvergenceChecker;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import sun.tools.jar.resources.jar;
import java.io.*;
import java.util.*;
/**
* Created by Charith Wickramaarachchi on 6/30/14.
*/
public class ReduceCommunity extends Reducer<Text, BytesWritable, Text, Text> {
private double precision = 0.000001;
private boolean verbose;
int display_level = -1;
String outpath;
@Override
protected void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
Graph g = null;
try {
System.out.println("********************REDUCE****************");
g = reconstructGraph(values);
} catch (Exception e) {
e.printStackTrace();
throw new InterruptedException(e.toString());
}
System.out.println("REDUCE...... GRAPH DONE");
Community c = new Community(g, -1, precision);
double mod = c.modularity();
boolean improvement = true;
int level = 2;
if (verbose) {
System.out.print("level " + level);
System.out.print(" start computation");
System.out.println(" network size: "
+ c.getG().getNb_nodes() + " nodes, "
+ c.getG().getNb_links() + " links, "
+ c.getG().getTotal_weight() + " weight.");
}
improvement = c.one_level();
double new_mod = c.modularity();
if (++level == display_level)
g.display();
if (display_level == -1) {
String filePath = outpath + File.separator + "out_" + (level-1) + "_r.txt";
try {
c.display_partition(filePath);
} catch (Exception e) {
e.printStackTrace();
throw new InterruptedException(e.toString());
}
}
Graph g2 = c.partition2graph_binary();
if (verbose) {
System.out.println(" network size: "
+ c.getG().getNb_nodes() + " nodes, "
+ c.getG().getNb_links() + " links, "
+ c.getG().getTotal_weight() + " weight.");
}
c = new Community(g2, -1, precision);
mod = new_mod;
do {
if (verbose) {
System.out.print("level in loop" + level);
System.out.print(" start computation");
System.out.println(" network size: "
+ c.getG().getNb_nodes() + " nodes, "
+ c.getG().getNb_links() + " links, "
+ c.getG().getTotal_weight() + " weight.");
}
improvement = c.one_level();
new_mod = c.modularity();
if (++level == display_level)
g.display();
if (display_level == -1) {
String filePath = outpath + File.separator + "out_" + (level-1) + "_r.txt";
try {
c.display_partition(filePath);
} catch (Exception e) {
e.printStackTrace();
throw new InterruptedException(e.toString());
}
}
g2 = c.partition2graph_binary();
if (verbose) {
System.out.print(" network size: "
+ c.getG().getNb_nodes() + " nodes, "
+ c.getG().getNb_links() + " links, "
+ c.getG().getTotal_weight() + " weight.");
}
c = new Community(g2, -1, precision);
mod = new_mod;
} while (improvement);
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
Configuration configuration = context.getConfiguration();
verbose = configuration.getBoolean(LouvainMR.VERBOSE, false);
precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
this.outpath = configuration.get(LouvainMR.OUT_PATH);
System.out.println("verbose = " + verbose);
System.out.println("display_level = " + display_level);
System.out.println("outpath = " + outpath);
super.setup(context);
}
private Graph reconstructGraph(Iterable<BytesWritable> values) throws Exception {
Iterator<BytesWritable> it = values.iterator();
SortedMap<Integer, GraphMessage> map = new TreeMap<Integer, GraphMessage>();
//Load data
while (it.hasNext()) {
BytesWritable bytesWritable = it.next();
ByteArrayInputStream inputStream = new ByteArrayInputStream(bytesWritable.getBytes());
try {
ObjectInputStream objectInputStream = new ObjectInputStream(inputStream);
GraphMessage msg = (GraphMessage) objectInputStream.readObject();
map.put(msg.getCurrentPartition(), msg);
} catch (IOException e) {
e.printStackTrace();
throw new Exception(e);
}
}
// Renumber
int gap = 0;
int degreeGap = 0;
Path pt = new Path(outpath + File.separator +"Map-Partition-Sizes");
FileSystem fs = FileSystem.get(new Configuration());
if (fs.exists(pt)) {
fs.delete(pt, true);
}
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt, true)));
PrintWriter out = new PrintWriter(br);
for (int i = 0; i < map.keySet().size(); i++) {
GraphMessage msg = map.get(i);
long currentDegreelen = msg.getDegrees()[msg.getDegrees().length -1];
if (i != 0) {
for (int j = 0; j < msg.getLinks().length; j++) {
msg.getLinks()[j] += gap;
}
for (int j = 0; j < msg.getRemoteMap().length; j++) {
msg.getRemoteMap()[j].source += gap;
}
for (int j = 0; j < msg.getN2c().length; j++) {
msg.getN2c()[j] += gap;
}
for (int j = 0; j < msg.getDegrees().length; j++) {
msg.getDegrees()[j] += degreeGap;
}
}
out.println("" + i + "," + msg.getNb_nodes());
gap += msg.getNb_nodes();
degreeGap += currentDegreelen;
}
out.flush();
out.close();
//Integrate
Graph graph = new Graph();
for (int i = 0; i < map.keySet().size(); i++) {
GraphMessage msg = map.get(i);
Collections.addAll(graph.getDegrees().getList(), msg.getDegrees());
Collections.addAll(graph.getLinks().getList(), msg.getLinks());
Collections.addAll(graph.getWeights().getList(), msg.getWeights());
graph.setNb_links(graph.getNb_links() + msg.getNb_links());
graph.setNb_nodes((int) (graph.getNb_nodes() + msg.getNb_nodes()));
graph.setTotal_weight(graph.getTotal_weight() + msg.getTotal_weight());
}
//Merge local done.
Map<Integer,Vector<Integer>> remoteEdges = new HashMap<Integer, Vector<Integer>>();
Map<Integer,Vector<Float>> remoteWeighs = new HashMap<Integer, Vector<Float>>();
for(int i=0; i < map.keySet().size(); i++) {
Map<HashMap.SimpleEntry<Integer,Integer>, Float> m = new HashMap<AbstractMap.SimpleEntry<Integer, Integer>, Float>();
GraphMessage msg = map.get(i);
for (int j = 0; j < msg.getRemoteMap().length; j++) {
RemoteMap remoteMap = msg.getRemoteMap()[j];
int sink = remoteMap.sink;
int sinkPart = remoteMap.sinkPart;
int target = map.get(sinkPart).getN2c()[sink];
HashMap.SimpleEntry<Integer,Integer> key = new HashMap.SimpleEntry<Integer,Integer>(remoteMap.source,target);
if(m.containsKey(key)) {
m.put(key,m.get(key) + 1.0f);
} else {
m.put(key,1.0f);
}
}
graph.setNb_links(graph.getNb_links() + m.size());
Iterator<HashMap.SimpleEntry<Integer,Integer>> itr = m.keySet().iterator();
while (itr.hasNext()) {
HashMap.SimpleEntry<Integer,Integer> key = itr.next();
float w = m.get(key);
if(remoteEdges.containsKey(key.getKey())) {
remoteEdges.get(key.getKey()).getList().add(key.getValue());
if (remoteWeighs.containsKey(key.getKey())) {
remoteWeighs.get(key.getKey()).getList().add(w);
}
} else {
Vector<Integer> list = new Vector<Integer>();
list.getList().add(key.getValue());
remoteEdges.put(key.getKey(),list);
Vector<Float> wList = new Vector<Float>();
wList.getList().add(w);
remoteWeighs.put(key.getKey(),wList);
}
}
}
graph.addRemoteEdges(remoteEdges,remoteWeighs);
//Merge Remote Done
return graph;
}
}
