/*
TagRecommender:
A framework to implement and evaluate algorithms for the recommendation
of tags.
Copyright (C) 2013 Dominik Kowald
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package processing;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Timer;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
import com.google.common.primitives.Ints;
import common.DoubleMapComparator;
import common.Bookmark;
import common.MemoryThread;
import common.PerformanceMeasurement;
import common.Utilities;
import cc.mallet.pipe.StringList2FeatureSequence;
import cc.mallet.topics.ParallelTopicModel;
import cc.mallet.types.Alphabet;
import cc.mallet.types.IDSorter;
import cc.mallet.types.Instance;
import cc.mallet.types.InstanceList;
import file.BookmarkWriter;
import file.PredictionFileWriter;
import file.BookmarkReader;
import file.BookmarkSplitter;
public class MalletCalculator {
private final static int MAX_RECOMMENDATIONS = 10;
private final static int MAX_TERMS = 100;
//private final static int NUM_THREADS = 10;
private final static int NUM_ITERATIONS = 2000;
private final static double ALPHA = 0.01;
private final static double BETA = 0.01;
private static double TOPIC_THRESHOLD = 0.001;
private int numTopics;
private List<Map<Integer, Integer>> maps;
private InstanceList instances;
private List<Map<Integer, Double>> docList;
private List<Map<Integer, Double>> topicList;
private Map<Integer, Double> mostPopularTopics;
public MalletCalculator(List<Map<Integer, Integer>> maps, int numTopics) {
this.numTopics = numTopics;
this.maps = maps;
this.mostPopularTopics = new LinkedHashMap<Integer, Double>();
initializeDataStructures();
}
private void initializeDataStructures() {
this.instances = new InstanceList(new StringList2FeatureSequence());
for (Map<Integer, Integer> map : this.maps) {
List<String> tags = new ArrayList<String>();
for (Map.Entry<Integer, Integer> entry : map.entrySet()) {
for (int i = 0; i < entry.getValue(); i++) {
tags.add(entry.getKey().toString());
}
}
Instance inst = new Instance(tags, null, null, null);
inst.setData(tags);
this.instances.addThruPipe(inst);
}
}
private List<Map<Integer, Double>> getMaxTopicsByDocs(ParallelTopicModel LDA, int maxTopicsPerDoc) {
List<Map<Integer, Double>> docList = new ArrayList<Map<Integer, Double>>();
Map<Integer, Double> unsortedMostPopularTopics = new LinkedHashMap<Integer, Double>();
int numDocs = this.instances.size();
for (int doc = 0; doc < numDocs; ++doc) {
Map<Integer, Double> topicList = new LinkedHashMap<Integer, Double>();
double[] topicProbs = LDA.getTopicProbabilities(doc);
//double probSum = 0.0;
for (int topic = 0; topic < topicProbs.length && topic < maxTopicsPerDoc; topic++) {
if (topicProbs[topic] > TOPIC_THRESHOLD) { // TODO
double newTopicProb = topicProbs[topic];
topicList.put(topic, newTopicProb);
Double oldTopicProb = unsortedMostPopularTopics.get(topic);
unsortedMostPopularTopics.put(topic, oldTopicProb == null ? newTopicProb : oldTopicProb.doubleValue() + newTopicProb);
//probSum += topicProbs[topic];
}
}
//System.out.println("Topic Sum: " + probSum);
Map<Integer, Double> sortedTopicList = new TreeMap<Integer, Double>(new DoubleMapComparator(topicList));
sortedTopicList.putAll(topicList);
docList.add(sortedTopicList);
}
Map<Integer, Double> sortedMostPopularTopics = new TreeMap<Integer, Double>(new DoubleMapComparator(unsortedMostPopularTopics));
sortedMostPopularTopics.putAll(unsortedMostPopularTopics);
for (Map.Entry<Integer, Double> entry : sortedMostPopularTopics.entrySet()) {
if (this.mostPopularTopics.size() < MAX_RECOMMENDATIONS) {
this.mostPopularTopics.put(entry.getKey(), entry.getValue());
}
}
return docList;
}
private List<Map<Integer, Double>> getMaxTermsByTopics(ParallelTopicModel LDA, int limit) {
Alphabet alphabet = LDA.getAlphabet();
List<Map<Integer, Double>> topicList = new ArrayList<Map<Integer, Double>>();
int numTopics = LDA.getNumTopics();
List<TreeSet<IDSorter>> sortedWords = LDA.getSortedWords();
for (int topic = 0; topic < numTopics; ++topic) {
Map<Integer, Double> termList = new LinkedHashMap<Integer, Double>();
TreeSet<IDSorter> topicWords = sortedWords.get(topic);
//int i = 0;
double weightSum = 0.0;
for (IDSorter entry : topicWords) {
if (entry.getWeight() > 0.0) {
//if (i++ < limit) {
int tag = Integer.parseInt(alphabet.lookupObject(entry.getID()).toString());
termList.put(tag, entry.getWeight());
weightSum += entry.getWeight();
//} else {
// break;
//}
}
}
// relative values
//double relSum = 0.0;
for (Map.Entry<Integer, Double> entry : termList.entrySet()) {
//relSum += (entry.getValue() / weightSum);
entry.setValue(entry.getValue() / weightSum);
}
//System.out.println("RelSum: " + relSum);
topicList.add(termList);
}
return topicList;
}
public void predictValuesProbs(boolean topicCreation) {
ParallelTopicModel LDA = new ParallelTopicModel(this.numTopics, ALPHA * this.numTopics, BETA); // TODO
LDA.addInstances(this.instances);
LDA.setNumThreads(1);
LDA.setNumIterations(NUM_ITERATIONS);
LDA.setRandomSeed(43);
try {
LDA.estimate();
} catch (Exception e) {
e.printStackTrace();
}
this.docList = getMaxTopicsByDocs(LDA, this.numTopics);
System.out.println("Fetched Doc-List");
this.topicList = !topicCreation ? getMaxTermsByTopics(LDA, MAX_TERMS) : null;
System.out.println("Fetched Topic-List");
}
public Map<Integer, Double> getValueProbsForID(int id, boolean topicCreation) {
Map<Integer, Double> terms = null;
if (id < this.docList.size()) {
Map<Integer, Double> docVals = this.docList.get(id);
if (this.topicList == null) {
return docVals;
}
terms = new LinkedHashMap<Integer, Double>();
for (Map.Entry<Integer, Double> topic : docVals.entrySet()) { // look at each assigned topic
Set<Entry<Integer, Double>> entrySet = this.topicList.get(topic.getKey()).entrySet();
double topicProb = topic.getValue();
for (Map.Entry<Integer, Double> entry : entrySet) { // and its terms
if (topicCreation) {
// DEPRECATED
if (topicProb > TOPIC_THRESHOLD) {
terms.put(entry.getKey(), topicProb);
break; // only use first tag as topic-name with the topic probability
}
} else {
double wordProb = entry.getValue();
Double val = terms.get(entry.getKey());
terms.put(entry.getKey(), val == null ? wordProb * topicProb : val + wordProb * topicProb);
}
}
}
}
return terms;
}
public Map<Integer, Double> getMostPopularTopics() {
return this.mostPopularTopics;
}
// Statics -------------------------------------------------------------------------------------------------------------------------
private static String timeString;
private static Map<Integer, Double> getRankedTagList(BookmarkReader reader, Map<Integer, Double> userMap, Map<Integer, Double> resMap, boolean sorting) {
Map<Integer, Double> resultMap = new LinkedHashMap<Integer, Double>();
if (userMap != null) {
for (Map.Entry<Integer, Double> entry : userMap.entrySet()) {
resultMap.put(entry.getKey(), entry.getValue().doubleValue());
}
}
if (resMap != null) {
for (Map.Entry<Integer, Double> entry : resMap.entrySet()) {
double resVal = entry.getValue().doubleValue();
Double val = resultMap.get(entry.getKey());
resultMap.put(entry.getKey(), val == null ? resVal : val.doubleValue() + resVal);
}
}
if (sorting) {
Map<Integer, Double> sortedResultMap = new TreeMap<Integer, Double>(new DoubleMapComparator(resultMap));
sortedResultMap.putAll(resultMap);
Map<Integer, Double> returnMap = new LinkedHashMap<Integer, Double>(MAX_RECOMMENDATIONS);
int i = 0;
for (Map.Entry<Integer, Double> entry : sortedResultMap.entrySet()) {
if (i++ < MAX_RECOMMENDATIONS) {
returnMap.put(entry.getKey(), entry.getValue());
} else {
break;
}
}
return returnMap;
}
return resultMap;
}
private static List<Map<Integer, Double>> startLdaCreation(BookmarkReader reader, int sampleSize, boolean sorting, int numTopics, boolean userBased, boolean resBased, boolean topicCreation) {
int size = reader.getBookmarks().size();
int trainSize = size - sampleSize;
//int oldTrainSize = trainSize;
Stopwatch timer = new Stopwatch();
timer.start();
MalletCalculator userCalc = null;
List<Map<Integer, Integer>> userMaps = null;
if (userBased) {
userMaps = Utilities.getUserMaps(reader.getBookmarks().subList(0, trainSize));
userCalc = new MalletCalculator(userMaps, numTopics);
userCalc.predictValuesProbs(topicCreation);
System.out.println("User-Training finished");
}
MalletCalculator resCalc = null;
List<Map<Integer, Integer>> resMaps = null;
if (resBased) {
resMaps = Utilities.getResMaps(reader.getBookmarks().subList(0, trainSize));
resCalc = new MalletCalculator(resMaps, numTopics);
resCalc.predictValuesProbs(topicCreation);
System.out.println("Res-Training finished");
}
List<Map<Integer, Double>> results = new ArrayList<Map<Integer, Double>>();
if (topicCreation) {
trainSize = 0;
}
timer.stop();
long trainingTime = timer.elapsed(TimeUnit.MILLISECONDS);
timer.reset();
timer.start();
int mpCount = 0;
for (int i = trainSize; i < size; i++) { // the test set
Bookmark data = reader.getBookmarks().get(i);
int userID = data.getUserID();
int resID = data.getResourceID();
Map<Integer, Double> userPredMap = null;
if (userCalc != null) {
userPredMap = userCalc.getValueProbsForID(userID, topicCreation);
}
Map<Integer, Double> resPredMap = null;
if (resCalc != null) {
//if (i > oldTrainSize) {
// System.out.println("Test-Set");
//}
resPredMap = resCalc.getValueProbsForID(resID, topicCreation);
if (topicCreation && resPredMap == null) {
resPredMap = resCalc.getMostPopularTopics();
mpCount++;
}
}
Map<Integer, Double> map = getRankedTagList(reader, userPredMap, resPredMap, sorting);
results.add(map);
}
timer.stop();
long testTime = timer.elapsed(TimeUnit.MILLISECONDS);
timeString = PerformanceMeasurement.addTimeMeasurement(timeString, true, trainingTime, testTime, (topicCreation ? size : sampleSize));
System.out.println("MpCount: " + mpCount);
return results;
}
// public statics ---------------------------------------------------------------------------------------------------------------------------------
public static BookmarkReader predictSample(String filename, int trainSize, int sampleSize, int numTopics, boolean userBased, boolean resBased) {
Timer timerThread = new Timer();
MemoryThread memoryThread = new MemoryThread();
timerThread.schedule(memoryThread, 0, MemoryThread.TIME_SPAN);
BookmarkReader reader = new BookmarkReader(trainSize, false);
reader.readFile(filename);
List<Map<Integer, Double>> ldaValues = startLdaCreation(reader, sampleSize, true, numTopics, userBased, resBased, false);
List<int[]> predictionValues = new ArrayList<int[]>();
for (int i = 0; i < ldaValues.size(); i++) {
Map<Integer, Double> ldaVal = ldaValues.get(i);
predictionValues.add(Ints.toArray(ldaVal.keySet()));
}
reader.setTestLines(reader.getBookmarks().subList(trainSize, reader.getBookmarks().size()));
PredictionFileWriter writer = new PredictionFileWriter(reader, predictionValues);
writer.writeFile(filename + "_lda_" + numTopics);
timeString = PerformanceMeasurement.addMemoryMeasurement(timeString, false, memoryThread.getMaxMemory());
timerThread.cancel();
Utilities.writeStringToFile("./data/metrics/" + filename + "_lda_" + numTopics + "_TIME.txt", timeString);
return reader;
}
public static void createSample(String filename, short numTopics, boolean tagRec, int trainSize, boolean personalizedTopicCreation) {
Timer timerThread = new Timer();
MemoryThread memoryThread = new MemoryThread();
timerThread.schedule(memoryThread, 0, MemoryThread.TIME_SPAN);
String outputFile = new String(filename) + "_lda_" + numTopics;
if (tagRec) {
TOPIC_THRESHOLD = 0.001;
} else {
TOPIC_THRESHOLD = 0.01;
}
Integer creationTrainSize = (personalizedTopicCreation ? trainSize : null);
BookmarkReader reader = new BookmarkReader(creationTrainSize == null ? 0 : creationTrainSize.intValue(), false);
reader.readFile(filename);
int size = reader.getBookmarks().size();
List<Map<Integer, Double>> ldaValues = startLdaCreation(reader, creationTrainSize == null ? 0 : size - creationTrainSize.intValue(), true, numTopics, false, true, true);
List<int[]> predictionValues = new ArrayList<int[]>();
//List<double[]> probValues = new ArrayList<double[]>();
for (int i = 0; i < ldaValues.size(); i++) {
Map<Integer, Double> ldaVal = ldaValues.get(i);
predictionValues.add(Ints.toArray(ldaVal.keySet()));
//probValues.add(Doubles.toArray(ldaVal.values()));
}
List<Bookmark> userSample = reader.getBookmarks().subList(0, size);
BookmarkWriter.writeSample(reader, userSample, outputFile, predictionValues, false);
//if (creationTrainSize != null) {
List<Bookmark> trainUserSample = reader.getBookmarks().subList(0, trainSize);
List<int[]> trainPredictionValues = predictionValues.subList(0, trainSize);
List<Bookmark> testUserSample = reader.getBookmarks().subList(trainSize, size);
List<int[]> testPredictionValues = predictionValues.subList(trainSize, size);
BookmarkWriter.writeSample(reader, trainUserSample, outputFile + "_train", trainPredictionValues, false);
BookmarkWriter.writeSample(reader, testUserSample, outputFile + "_test", testPredictionValues, false);
//}
timeString = PerformanceMeasurement.addMemoryMeasurement(timeString, false, memoryThread.getMaxMemory());
timerThread.cancel();
Utilities.writeStringToFile("./data/metrics/" + filename + "_lda_creation_" + numTopics + "_TIME.txt", timeString);
}
}
