/*******************************************************************************
* Copyright (c) 2011 Dipanjan Das
* Language Technologies Institute,
* Carnegie Mellon University,
* All Rights Reserved.
*
* LinDekNeighbors.java is part of SEMAFOR 2.0.
*
* SEMAFOR 2.0 is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* SEMAFOR 2.0 is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with SEMAFOR 2.0. If not, see <http://www.gnu.org/licenses/>.
******************************************************************************/
package edu.cmu.cs.lti.ark.fn.identification;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.StringTokenizer;
import edu.cmu.cs.lti.ark.fn.data.prep.ParsePreparation;
import edu.cmu.cs.lti.ark.fn.wordnet.WordNetRelations;
import edu.cmu.cs.lti.ark.util.ds.Pair;
import gnu.trove.THashSet;
import gnu.trove.TObjectIntHashMap;
public class LinDekNeighbors {
public static final int MAX_NEIGHBORS = 20;
public static void main(String[] args) {
Pair<TObjectIntHashMap<String>, TObjectIntHashMap<String>> p =
readAdjectivesAndAdverbs();
TObjectIntHashMap<String> adjectives = p.getFirst();
TObjectIntHashMap<String> adverbs = p.getSecond();
System.out.println("Number of adjectives:" + adjectives.size());
System.out.println("Number of adverbs:" + adverbs.size());
try {
//createNeighborsForAdverbsAndAdjectives(p);
//createNeighborsForNouns();
createNeighborsForVerbs();
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
}
public static void createNeighborsForVerbs()
throws IOException {
String stopfile = "lrdata/stopwords.txt";
String wnConfigFile = "file_properties.xml";
WordNetRelations wnr = new WordNetRelations(stopfile, wnConfigFile);
System.out.println(wnr.getLemmaForWord("worse", "J"));
if (true)
System.exit(-1);
String lindekdirectory = "/home/dipanjan/work/fall2010/SSL/FNData";
String outFile = "/home/dipanjan/work/fall2010/SSL/FNData/lindekneighbors.dat";
BufferedWriter bWriter = new BufferedWriter(new FileWriter(outFile, true));
String line;
BufferedReader bReader = new BufferedReader(new FileReader(lindekdirectory + "/simV.lsp"));
line = bReader.readLine();
ArrayList<String> lines = new ArrayList<String>();
while (line != null) {
line = line.trim();
lines.clear();
while (!line.equals("))")) {
lines.add(line);
line = bReader.readLine();
}
String firstLine = lines.get(0);
firstLine = firstLine.substring(1);
int ind = firstLine.indexOf("(desc");
firstLine = firstLine.substring(0, ind).trim();
String pos = null;
// multiword case
boolean isAdjective = false;
boolean isAdverb = false;
int knn = 0;
String outline = "";
for (int i = 1; i < lines.size() ; i ++) {
StringTokenizer st = new StringTokenizer(lines.get(i).trim(), " \t", true);
ArrayList<String> toks = new ArrayList<String>();
while (st.hasMoreTokens()) {
toks.add(st.nextToken());
}
double value = new Double(toks.get(toks.size()-1));
String unit = "";
for (int j = 0; j < toks.size()-2; j++) {
unit += toks.get(j);
}
if (unit.startsWith("\"")) {
if (!unit.endsWith("\"")) {
System.out.println("Problem with unit:" + unit);
System.exit(-1);
}
unit = unit.substring(1, unit.length()-1).toLowerCase();
} else {
String lc = unit.toLowerCase();
lc = wnr.getLemmaForWord(lc, "V");
unit = lc;
}
outline += unit + ".v\t" + value +"\t";
knn++;
if (knn > MAX_NEIGHBORS) {
break;
}
}
outline = outline.trim();
if (firstLine.startsWith("\"")) {
if (!firstLine.endsWith("\"")) {
System.out.println("Problem with unit:" + firstLine);
System.exit(-1);
}
firstLine = firstLine.substring(1, firstLine.length()-1).toLowerCase();
} else {
String lc = firstLine.toLowerCase();
firstLine = wnr.getLemmaForWord(lc, "V");
}
bWriter.write(firstLine + ".v\t" + outline + "\n");
line = bReader.readLine();
}
bReader.close();
bWriter.close();
}
public static void createNeighborsForNouns()
throws IOException {
String stopfile = "lrdata/stopwords.txt";
String wnConfigFile = "file_properties.xml";
WordNetRelations wnr = new WordNetRelations(stopfile, wnConfigFile);
String lindekdirectory = "/home/dipanjan/work/fall2010/SSL/FNData";
String outFile = "/home/dipanjan/work/fall2010/SSL/FNData/lindekneighbors.dat";
BufferedWriter bWriter = new BufferedWriter(new FileWriter(outFile, true));
String line;
BufferedReader bReader = new BufferedReader(new FileReader(lindekdirectory + "/simN.lsp"));
line = bReader.readLine();
ArrayList<String> lines = new ArrayList<String>();
while (line != null) {
line = line.trim();
lines.clear();
while (!line.equals("))")) {
lines.add(line);
line = bReader.readLine();
}
String firstLine = lines.get(0);
firstLine = firstLine.substring(1);
int ind = firstLine.indexOf("(desc");
firstLine = firstLine.substring(0, ind).trim();
String pos = null;
// multiword case
boolean isAdjective = false;
boolean isAdverb = false;
int knn = 0;
String outline = "";
for (int i = 1; i < lines.size() ; i ++) {
StringTokenizer st = new StringTokenizer(lines.get(i).trim(), " \t", true);
ArrayList<String> toks = new ArrayList<String>();
while (st.hasMoreTokens()) {
toks.add(st.nextToken());
}
double value = new Double(toks.get(toks.size()-1));
String unit = "";
for (int j = 0; j < toks.size()-2; j++) {
unit += toks.get(j);
}
if (unit.startsWith("\"")) {
if (!unit.endsWith("\"")) {
System.out.println("Problem with unit:" + unit);
System.exit(-1);
}
unit = unit.substring(1, unit.length()-1).toLowerCase();
} else {
String lc = unit.toLowerCase();
lc = wnr.getLemmaForWord(lc, "N");
unit = lc;
}
outline += unit + ".n\t" + value +"\t";
knn++;
if (knn > MAX_NEIGHBORS) {
break;
}
}
outline = outline.trim();
if (firstLine.startsWith("\"")) {
if (!firstLine.endsWith("\"")) {
System.out.println("Problem with unit:" + firstLine);
System.exit(-1);
}
firstLine = firstLine.substring(1, firstLine.length()-1).toLowerCase();
} else {
String lc = firstLine.toLowerCase();
firstLine = wnr.getLemmaForWord(lc, "N");
}
bWriter.write(firstLine + ".n\t" + outline + "\n");
line = bReader.readLine();
}
bReader.close();
bWriter.close();
}
public static void createNeighborsForAdverbsAndAdjectives(Pair<TObjectIntHashMap<String>,
TObjectIntHashMap<String>> p)
throws IOException {
String lindekdirectory = "/home/dipanjan/work/fall2010/SSL/FNData";
String outFile = "/home/dipanjan/work/fall2010/SSL/FNData/lindekneighbors.dat";
BufferedWriter bWriter = new BufferedWriter(new FileWriter(outFile));
String line;
ArrayList<String> mAdjectives = ParsePreparation.readSentencesFromFile(lindekdirectory + "/mult.word.j");
THashSet<String> adjSet = new THashSet<String>();
for (int i = 0; i < mAdjectives.size(); i++) {
String w = mAdjectives.get(i).toLowerCase();
adjSet.add(w);
}
ArrayList<String> mAdverbs = ParsePreparation.readSentencesFromFile(lindekdirectory + "/mult.word.r");
THashSet<String> advSet = new THashSet<String>();
for (int i = 0; i < mAdverbs.size(); i++) {
String w = mAdverbs.get(i).toLowerCase();
advSet.add(w);
}
TObjectIntHashMap<String> dAdjectives = p.getFirst();
TObjectIntHashMap<String> dAdverbs = p.getSecond();
// adverbs and adjectives;
BufferedReader bReader = new BufferedReader(new FileReader(lindekdirectory + "/simA.lsp"));
line = bReader.readLine();
ArrayList<String> lines = new ArrayList<String>();
while (line != null) {
line = line.trim();
lines.clear();
while (!line.equals("))")) {
lines.add(line);
line = bReader.readLine();
}
String firstLine = lines.get(0);
firstLine = firstLine.substring(1);
int ind = firstLine.indexOf("(desc");
firstLine = firstLine.substring(0, ind).trim();
// multiword case
boolean isAdjective = false;
boolean isAdverb = false;
int knn = 0;
String outline = "";
for (int i = 1; i < lines.size() ; i ++) {
StringTokenizer st = new StringTokenizer(lines.get(i).trim(), " \t", true);
ArrayList<String> toks = new ArrayList<String>();
while (st.hasMoreTokens()) {
toks.add(st.nextToken());
}
double value = new Double(toks.get(toks.size()-1));
String unit = "";
for (int j = 0; j < toks.size()-2; j++) {
unit += toks.get(j);
}
if (unit.startsWith("\"")) {
if (!unit.endsWith("\"")) {
System.out.println("Problem with unit:" + unit);
System.exit(-1);
}
unit = unit.substring(1, unit.length()-1).toLowerCase();
isAdjective = isMultiWordAdjective(adjSet, unit);
isAdverb = isMultiWordAdverb(advSet, unit);
} else {
String lc = unit.toLowerCase();
int wpos = findWordPOS(dAdjectives,
dAdverbs,
lc);
if (wpos == 1) {
isAdjective = true;
} else if (wpos == 2) {
isAdverb = true;
} else {
isAdjective = true;
isAdverb = true;
}
unit = lc;
}
if (isAdjective)
outline += unit + ".a\t" + value +"\t";
if (isAdverb)
outline += unit + ".adv\t" + value +"\t";
knn++;
if (knn > MAX_NEIGHBORS) {
break;
}
}
outline = outline.trim();
if (firstLine.startsWith("\"")) {
if (!firstLine.endsWith("\"")) {
System.out.println("Problem with unit:" + firstLine);
System.exit(-1);
}
firstLine = firstLine.substring(1, firstLine.length()-1).toLowerCase();
isAdjective = isMultiWordAdjective(adjSet, firstLine);
isAdverb = isMultiWordAdverb(advSet, firstLine);
} else {
String lc = firstLine.toLowerCase();
int wpos = findWordPOS(dAdjectives,
dAdverbs,
lc);
if (wpos == 1) {
isAdjective = true;
} else if (wpos == 2) {
isAdverb = true;
} else {
isAdjective = true;
isAdverb = true;
}
firstLine = lc;
}
if (isAdjective) {
bWriter.write(firstLine + ".a\t" + outline + "\n");
}
if (isAdverb) {
bWriter.write(firstLine + ".adv\t" + outline + "\n");
}
line = bReader.readLine();
}
bReader.close();
bWriter.close();
}
public static boolean isMultiWordAdjective(THashSet<String> mAdjectives,
String word) {
return mAdjectives.contains(word);
}
public static boolean isMultiWordAdverb(THashSet<String> mAdverbs,
String word) {
return mAdverbs.contains(word);
}
// 1 : adjective
// 2: adverb
// 3: both
public static int findWordPOS(TObjectIntHashMap<String> dAdjectives,
TObjectIntHashMap<String> dAdverbs,
String word) {
int adjCount = dAdjectives.get(word);
int advCount = dAdverbs.get(word);
if (adjCount == 0 && advCount == 0) {
if (word.endsWith("ly"))
return 2;
else {
return 3;
}
}
int total = adjCount + advCount;
double adjProb = (double)adjCount / (double) total;
double advProb = (double)advCount / (double) total;
if (Math.abs(adjProb - advProb) < 0.2) {
return 3;
} else {
if (adjProb > advProb) {
return 1;
} else {
return 2;
}
}
}
public static Pair<TObjectIntHashMap<String>, TObjectIntHashMap<String>>
readAdjectivesAndAdverbs() {
String adjFile = "/home/dipanjan/work/fall2010/SSL/FNData/gw.a";
String advFile = "/home/dipanjan/work/fall2010/SSL/FNData/gw.adv";
ArrayList<String> adjectives =
ParsePreparation.readSentencesFromFile(adjFile);
ArrayList<String> adverbs =
ParsePreparation.readSentencesFromFile(advFile);
TObjectIntHashMap<String> adjMap =
new TObjectIntHashMap<String>();
TObjectIntHashMap<String> advMap =
new TObjectIntHashMap<String>();
for (String string: adjectives) {
String[] toks = string.trim().split("\t");
adjMap.put(toks[0], new Integer(toks[1]));
}
for (String string: adverbs) {
String[] toks = string.trim().split("\t");
try {
advMap.put(toks[0], new Integer(toks[1]));
} catch (Exception e) {
System.out.println(string + "\n\n");
e.printStackTrace();
System.exit(-1);
}
}
return new Pair<TObjectIntHashMap<String>, TObjectIntHashMap<String>>(adjMap, advMap);
}
public static Pair<TObjectIntHashMap<String>, TObjectIntHashMap<String>>
scanAdjectivesAndAdverbs() {
String largeFile =
"/home/dipanjan/work/fall2010/SSL/FNData/AP_1m.all.lemma.tags";
TObjectIntHashMap<String> adjectives = new TObjectIntHashMap<String>();
TObjectIntHashMap<String> adverbs = new TObjectIntHashMap<String>();
try {
BufferedReader bReader = new BufferedReader(new FileReader(largeFile));
String line = null;
int count = 0;
while ((line = bReader.readLine()) != null) {
line = line.trim();
String[] toks = line.split("\t");
int numTokens = new Integer(toks[0]);
for (int i = 0; i < numTokens; i++) {
String pos = toks[1 + numTokens + i];
if (pos.startsWith("J")) {
int c = adjectives.get(toks[1 + i].toLowerCase());
adjectives.put(toks[1 + i].toLowerCase(), c+1);
} else if (pos.startsWith("RB")) {
int c = adverbs.get(toks[1 + i].toLowerCase());
adverbs.put(toks[1 + i].toLowerCase(), c+1);
}
}
count++;
if (count % 1000 == 0) {
System.out.print(". ");
}
if (count % 10000 == 0) {
System.out.println(count);
}
// if (count > 1000)
// break;
}
bReader.close();
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
return new Pair<TObjectIntHashMap<String>, TObjectIntHashMap<String>>(adjectives, adverbs);
}
}
