/*
* NGramCorrelationForm.java
*
* Created on 24 ??????? 2006, 1:43 ??
*/
package gr.demokritos.iit.jinsect.gui;
import gr.demokritos.iit.conceptualIndex.LocalWordNetMeaningExtractor;
import gr.demokritos.iit.conceptualIndex.structs.Concatenation;
import gr.demokritos.iit.conceptualIndex.documentModel.DistributionDocument;
import gr.demokritos.iit.conceptualIndex.documentModel.SemanticIndex;
import gr.demokritos.iit.conceptualIndex.documentModel.SymbolicGraph;
import gr.demokritos.iit.conceptualIndex.structs.Union;
import gr.demokritos.iit.conceptualIndex.structs.Distribution;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.StreamTokenizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import gr.demokritos.iit.jinsect.structs.CategorizedFileEntry;
import gr.demokritos.iit.jinsect.IMatching;
import gr.demokritos.iit.jinsect.structs.DocumentSet;
import gr.demokritos.iit.jinsect.algorithms.estimators.DistanceEstimator;
import gr.demokritos.iit.jinsect.algorithms.estimators.NGramSizeEstimator;
import gr.demokritos.iit.jinsect.algorithms.statistics.statisticalCalculation;
import gr.demokritos.iit.jinsect.console.StatusConsole;
import gr.demokritos.iit.jinsect.console.StreamOutputConsole;
import gr.demokritos.iit.jinsect.documentModel.documentTypes.NGramDocument;
import gr.demokritos.iit.jinsect.structs.IntegerPair;
import gr.demokritos.iit.jinsect.supportUtils.linguistic.WordDefinition;
import gr.demokritos.iit.jinsect.threading.ThreadList;
import gr.demokritos.iit.jinsect.utils;
import gr.demokritos.iit.summarization.analysis.EntropyChunker;
import salvo.jesus.graph.Vertex;
import salvo.jesus.graph.VertexImpl;
import salvo.jesus.graph.WeightedEdge;
/**
*
* @author ggianna
*/
public class NGramCorrelationForm extends javax.swing.JFrame implements IMatching {
DistributionDocument[] cdDoc;
SymbolicGraph sgOverallGraph;
SemanticIndex siIndex;
private Date dLastUpdate = new Date();
ArrayList lActiveActions = new ArrayList();
TreeMap Delims = new TreeMap();
EntropyChunker Chunker = new EntropyChunker();
boolean RightToLeftText = false;
public static final int MinLevel = 0;
//public static final int Levels = 10;
/** Creates new form NGramCorrelationForm */
public NGramCorrelationForm() {
initComponents();
}
/** This method is called from within the constructor to
* initialize the form.
* WARNING: Do NOT modify this code. The content of this method is
* always regenerated by the Form Editor.
*/
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
java.awt.GridBagConstraints gridBagConstraints;
FilePathEdt = new javax.swing.JTextField();
SelectInputFileBtn = new javax.swing.JButton();
CreateNGramGraphBtn = new javax.swing.JButton();
NGramSizeSldr = new javax.swing.JSlider();
jLabel1 = new javax.swing.JLabel();
jScrollPane1 = new javax.swing.JScrollPane();
LogMemo = new javax.swing.JTextArea();
jLabel2 = new javax.swing.JLabel();
TermEdt = new javax.swing.JTextField();
jLabel3 = new javax.swing.JLabel();
LookUpTermBtn = new javax.swing.JButton();
BreakDownBtn = new javax.swing.JButton();
FindPathBtn = new javax.swing.JButton();
jLabel4 = new javax.swing.JLabel();
CorpusPercentSld = new javax.swing.JSlider();
ClearGraphsBtn = new javax.swing.JButton();
jLabel5 = new javax.swing.JLabel();
SelectInputFileEdt = new javax.swing.JTextField();
SelectTestFileBtn = new javax.swing.JButton();
BreakFileDownBtn = new javax.swing.JButton();
jButton1 = new javax.swing.JButton();
SecondInputFileLbl = new javax.swing.JLabel();
SelectSecondInputFileEdt = new javax.swing.JTextField();
SelectSecondTestFileBtn = new javax.swing.JButton();
CompareFilesBtn = new javax.swing.JButton();
CancelAllBtn = new javax.swing.JButton();
AnalyseCorpusBtn = new javax.swing.JButton();
jLabel6 = new javax.swing.JLabel();
DelimitersEdt = new javax.swing.JTextField();
setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
setTitle("NGram Correlation Experiments");
setName("NGramCorrelationFrame"); // NOI18N
getContentPane().setLayout(new java.awt.GridBagLayout());
FilePathEdt.setText("/home/ggianna/Documents/JApplications/JInsect/conceptualCorpus");
FilePathEdt.setName("FilePathEd"); // NOI18N
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 1;
gridBagConstraints.gridy = 0;
gridBagConstraints.gridwidth = 4;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(FilePathEdt, gridBagConstraints);
SelectInputFileBtn.setText("Select input file");
SelectInputFileBtn.setName("SelectInputFileBtn"); // NOI18N
SelectInputFileBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
SelectInputFileBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 5;
gridBagConstraints.gridy = 0;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(SelectInputFileBtn, gridBagConstraints);
CreateNGramGraphBtn.setText("Create NGram Graphs");
CreateNGramGraphBtn.setName("CreateGraphBtn"); // NOI18N
CreateNGramGraphBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
CreateNGramGraphBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 14;
getContentPane().add(CreateNGramGraphBtn, gridBagConstraints);
NGramSizeSldr.setMajorTickSpacing(5);
NGramSizeSldr.setMaximum(22);
NGramSizeSldr.setMinimum(2);
NGramSizeSldr.setMinorTickSpacing(1);
NGramSizeSldr.setPaintLabels(true);
NGramSizeSldr.setPaintTicks(true);
NGramSizeSldr.setValue(9);
NGramSizeSldr.setName("NGramSizeSldr"); // NOI18N
NGramSizeSldr.setValueIsAdjusting(true);
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 1;
gridBagConstraints.gridy = 1;
gridBagConstraints.gridwidth = java.awt.GridBagConstraints.REMAINDER;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(NGramSizeSldr, gridBagConstraints);
jLabel1.setText("Max NGram Size");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 1;
getContentPane().add(jLabel1, gridBagConstraints);
jScrollPane1.setMaximumSize(null);
jScrollPane1.setMinimumSize(null);
jScrollPane1.setPreferredSize(new java.awt.Dimension(300, 400));
LogMemo.setColumns(20);
LogMemo.setRows(5);
LogMemo.setPreferredSize(null);
jScrollPane1.setViewportView(LogMemo);
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 6;
gridBagConstraints.gridwidth = java.awt.GridBagConstraints.REMAINDER;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(jScrollPane1, gridBagConstraints);
jLabel2.setText("Graph training corpus");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 0;
getContentPane().add(jLabel2, gridBagConstraints);
TermEdt.setMinimumSize(new java.awt.Dimension(200, 19));
TermEdt.setPreferredSize(new java.awt.Dimension(100, 19));
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 1;
gridBagConstraints.gridy = 3;
gridBagConstraints.gridwidth = 3;
getContentPane().add(TermEdt, gridBagConstraints);
jLabel3.setText("Select Term to Look up");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 3;
getContentPane().add(jLabel3, gridBagConstraints);
LookUpTermBtn.setText("Look up Term");
LookUpTermBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
LookUpTermBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 2;
gridBagConstraints.gridy = 5;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(LookUpTermBtn, gridBagConstraints);
BreakDownBtn.setText("Break term down");
BreakDownBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
BreakDownBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 5;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(BreakDownBtn, gridBagConstraints);
FindPathBtn.setText("Test Search");
FindPathBtn.setEnabled(false);
FindPathBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
FindPathBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 1;
gridBagConstraints.gridy = 5;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(FindPathBtn, gridBagConstraints);
jLabel4.setText("Select % of Corpus to Use");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 0;
gridBagConstraints.gridy = 2;
getContentPane().add(jLabel4, gridBagConstraints);
CorpusPercentSld.setMajorTickSpacing(10);
CorpusPercentSld.setMinorTickSpacing(5);
CorpusPercentSld.setPaintLabels(true);
CorpusPercentSld.setPaintTicks(true);
CorpusPercentSld.setValue(100);
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 1;
gridBagConstraints.gridy = 2;
gridBagConstraints.gridwidth = java.awt.GridBagConstraints.REMAINDER;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(CorpusPercentSld, gridBagConstraints);
ClearGraphsBtn.setText("Clear Graphs");
ClearGraphsBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
ClearGraphsBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 14;
getContentPane().add(ClearGraphsBtn, gridBagConstraints);
jLabel5.setText("Select Input File");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 8;
getContentPane().add(jLabel5, gridBagConstraints);
SelectInputFileEdt.setText("/home/ggianna/Documents/JApplications/JInsect/conceptualCorpus");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 8;
gridBagConstraints.gridwidth = 4;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(SelectInputFileEdt, gridBagConstraints);
SelectTestFileBtn.setText("Select test file");
SelectTestFileBtn.setName("SelectInputFileBtn"); // NOI18N
SelectTestFileBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
SelectTestFileBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 8;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(SelectTestFileBtn, gridBagConstraints);
BreakFileDownBtn.setText("Break file down");
BreakFileDownBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
BreakFileDownBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 8;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(BreakFileDownBtn, gridBagConstraints);
jButton1.setText("Chunk String");
jButton1.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton1ActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 4;
gridBagConstraints.gridy = 5;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(jButton1, gridBagConstraints);
SecondInputFileLbl.setText("Select Second File");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 9;
getContentPane().add(SecondInputFileLbl, gridBagConstraints);
SelectSecondInputFileEdt.setText("/home/ggianna/Documents/JApplications/JInsect/conceptualCorpus");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 9;
gridBagConstraints.gridwidth = 4;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(SelectSecondInputFileEdt, gridBagConstraints);
SelectSecondTestFileBtn.setText("Select second file");
SelectSecondTestFileBtn.setName("SelectInputFileBtn"); // NOI18N
SelectSecondTestFileBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
SelectSecondTestFileBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 9;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(SelectSecondTestFileBtn, gridBagConstraints);
CompareFilesBtn.setText("Compare Files");
CompareFilesBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
CompareFilesBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridx = 6;
gridBagConstraints.gridy = 9;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(CompareFilesBtn, gridBagConstraints);
CancelAllBtn.setText("Cancel All Processes");
CancelAllBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
CancelAllBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 15;
getContentPane().add(CancelAllBtn, gridBagConstraints);
AnalyseCorpusBtn.setText("Analyse Corpus");
AnalyseCorpusBtn.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
AnalyseCorpusBtnActionPerformed(evt);
}
});
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 14;
getContentPane().add(AnalyseCorpusBtn, gridBagConstraints);
jLabel6.setText("Delimiters");
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 13;
getContentPane().add(jLabel6, gridBagConstraints);
gridBagConstraints = new java.awt.GridBagConstraints();
gridBagConstraints.gridy = 13;
gridBagConstraints.gridwidth = 2;
gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
getContentPane().add(DelimitersEdt, gridBagConstraints);
pack();
}// </editor-fold>//GEN-END:initComponents
private void AnalyseCorpusBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_AnalyseCorpusBtnActionPerformed
Thread t = new Thread() {
@Override
public void run() {
DoAnalyseCorpus();
unregisterThread(this);
}
};
registerThread(t);
t.setPriority(Math.min(Thread.MAX_PRIORITY, t.getPriority() + 1)); // Increased priority
t.start();
}//GEN-LAST:event_AnalyseCorpusBtnActionPerformed
private synchronized void registerThread(Thread t) {
lActiveActions.add(t);
// DEBUG LINES
System.out.println("Added " + t.getId() + ":" + t.getName());
//////////////
}
private synchronized void unregisterThread(Thread t) {
lActiveActions.remove(t);
// DEBUG LINES
System.out.println("Removed " + t.getId() + ":" + t.getName());
//////////////
}
private void CancelAllBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_CancelAllBtnActionPerformed
if (JOptionPane.showConfirmDialog(this, "Are you sure you want to stop running processes?",
"Cancelling running processes", JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) {
Iterator iIter = lActiveActions.iterator();
while (iIter.hasNext()) {
Object oNext = iIter.next();
if (oNext instanceof Thread)
((Thread)oNext).interrupt();
iIter.remove();
}
}
}//GEN-LAST:event_CancelAllBtnActionPerformed
private List KeepLeastSizeSets(List lSets) {
return KeepLeastSizeSets(lSets, 0);
}
private List KeepLeastSizeSets(List lSets, int iMinContainedNGramSize) {
ArrayList lRes = new ArrayList();
// First pass - Get minimum size
int iMinSize = Integer.MAX_VALUE;
Iterator iIter = lSets.iterator();
while (iIter.hasNext()) {
List lNext = (List)iIter.next();
iMinSize = Math.min(iMinSize, lNext.size());
}
// Second pass - Keep only minimum-sized sets
iIter = lSets.iterator();
while (iIter.hasNext()) {
List lNext = (List)iIter.next();
if (lNext.size() == iMinSize)
lRes.add(lNext);
}
// Third pass - if needed - checking contained n-gram size
if (iMinContainedNGramSize > 0) {
iIter = lSets.iterator();
while (iIter.hasNext()) {
List lNext = (List)iIter.next();
boolean bIsOK = true;
Iterator iNGrams = lNext.iterator();
while (iNGrams.hasNext())
if (((String)iNGrams.next()).length() < iMinContainedNGramSize) {
bIsOK = false;
break;
}
if (!bIsOK)
iIter.remove();
}
}
return lRes;
}
private void DoCompareFiles() {
// First file
appendToLog("*** First file\n");
ArrayList Meanings1 = new ArrayList();
// Read file
String sDataString = "";
try {
ByteArrayOutputStream bsOut = new ByteArrayOutputStream();
FileInputStream fiIn = new FileInputStream(SelectInputFileEdt.getText());
int iData = 0;
while ((iData = fiIn.read()) > -1)
bsOut.write(iData);
sDataString = bsOut.toString();
}
catch (IOException ioe) {
ioe.printStackTrace();
}
// Chunk text
//String [] saSubtexts = chunkString(sDataString);
List<String> saSubtexts = Chunker.chunkString(sDataString);
//Iterator iStrIter = Arrays.asList(saSubtexts).iterator();
Iterator iStrIter = saSubtexts.iterator();
int iCnt = 0, iMax = saSubtexts.size();
final gr.demokritos.iit.jinsect.gui.StatusFrame fStatus =
new gr.demokritos.iit.jinsect.gui.StatusFrame();
fStatus.setVisible(true);
while (iStrIter.hasNext())
{
String sStr = (String)iStrIter.next();
// DEBUG LINES
// appendToLog("\nCHECKING\n" + sStr + "\n");
fStatus.setStatus("Checking: " + sStr, (double)iCnt++/iMax);
//////////////
// Analyse chunk in substrings
List lSubStrings = utils.getSubStrings(sStr, sStr.length(), this);
if (lSubStrings.size() == 0)
continue;
// Keep most promising substrings, i.e. with the minimal parts
// lSubStrings = KeepLeastSizeSets(lSubStrings, 2);
appendToLog(utils.printList(lSubStrings));
ArrayList lOptions = new ArrayList();
lOptions.addAll(lSubStrings);
Iterator iIter = lOptions.iterator();
HashMap hSubstringSet = new HashMap();
// Get meanings for every substring
while (iIter.hasNext()) {
Object oNext = iIter.next();
List lNext;
if (oNext instanceof List) {
lNext = (List)oNext;
}
else
{
lNext = new ArrayList();
lNext.add(oNext);
}
if (hSubstringSet.containsKey(lNext.toString()))
continue;
appendToLog("Case " + utils.printList(lNext));
hSubstringSet.put(lNext.toString(),1);
// Create vertex list
List lNodes = new ArrayList();
Iterator iSubstrings = lNext.iterator();
while (iSubstrings.hasNext()) {
lNodes.add(new VertexImpl(iSubstrings.next()));
}
// Attempt retrieval of union meanings
Iterator iNodes = lNodes.iterator();
String sUnionMeaning = "";
while (iNodes.hasNext()) {
String sCur = ((Vertex)iNodes.next()).toString();
Object oTxt = siIndex.getMeaning(new VertexImpl(sCur));
if (oTxt != null) {
sUnionMeaning += "-" + SemanticIndex.meaningToString(oTxt) + "-";
Meanings1.add(oTxt); // Add meaning to list
}
else
appendToLog("No meaning found...");
}
}
}
fStatus.setVisible(false);
// Second file
appendToLog("*** Second file\n");
ArrayList Meanings2 = new ArrayList();
// Read file
sDataString = "";
try {
ByteArrayOutputStream bsOut = new ByteArrayOutputStream();
FileInputStream fiIn = new FileInputStream(SelectSecondInputFileEdt.getText());
int iData = 0;
while ((iData = fiIn.read()) > -1)
bsOut.write(iData);
sDataString = bsOut.toString();
}
catch (IOException ioe) {
ioe.printStackTrace();
}
// Chunk text
//saSubtexts = Chunker.chunkString(sDataString);
saSubtexts = Chunker.chunkString(sDataString);
//iStrIter = Arrays.asList(saSubtexts).iterator();
iStrIter = saSubtexts.iterator();
while (iStrIter.hasNext())
{
String sStr = (String)iStrIter.next();
// DEBUG LINES
// appendToLog("\nCHECKING\n" + sStr + "\n");
//////////////
ArrayList lOptions = new ArrayList();
// Analyse chunk in substrings
List lSubStrings = utils.getSubStrings(sStr, sStr.length(), this, 2 * NGramSizeSldr.getValue());
// Keep most promising substrings, i.e. with the minimal parts
// lSubStrings = KeepLeastSizeSets(lSubStrings, 2);
appendToLog(utils.printList(lSubStrings));
lOptions.addAll(lSubStrings);
Iterator iIter = lOptions.iterator();
HashMap hSubstringSet = new HashMap();
// Get meanings for every substring
while (iIter.hasNext()) {
Object oNext = iIter.next();
List lNext;
if (oNext instanceof List) {
lNext = (List)oNext;
}
else
{
lNext = new ArrayList();
lNext.add(oNext);
}
if (hSubstringSet.containsKey(lNext.toString()))
continue;
appendToLog("Case " + utils.printList(lNext));
hSubstringSet.put(lNext.toString(),1);
// Create vertex list
List lNodes = new ArrayList();
Iterator iSubstrings = lNext.iterator();
while (iSubstrings.hasNext()) {
lNodes.add(new VertexImpl(iSubstrings.next()));
}
// Attempt retrieval of union meanings
Iterator iNodes = lNodes.iterator();
String sUnionMeaning = "";
while (iNodes.hasNext()) {
String sCur = ((Vertex)iNodes.next()).toString();
Object oTxt = siIndex.getMeaning(new VertexImpl(sCur));
if (oTxt != null) {
sUnionMeaning += "-" + siIndex.meaningToString(oTxt) + "-";
Meanings2.add(oTxt);
}
else
appendToLog("No meaning found...");
}
}
}
// Calc similarity
// TODO: DEBUG
// For every meaning in Meanings1
double dRes = 0.0;
Iterator iIter1 = Meanings1.iterator();
while (iIter1.hasNext()) {
double dMaxSim = 0.0;
Iterator iIter2 = Meanings2.iterator();
WordDefinition d1 = (WordDefinition)iIter1.next();
// For every meaning in Meanings2
while (iIter2.hasNext()) {
WordDefinition d2 = (WordDefinition)iIter2.next();
// Keep max similarity
dMaxSim = Math.max(dMaxSim, siIndex.compareWordDefinitions(d1, d2));
}
// DEBUG LINES
appendToLog("Concluded similarity of " + dMaxSim);
//////////////
dRes += dMaxSim;
}
dRes = 2 * dRes / (Meanings1.size() + Meanings2.size());
appendToLog("Final Similarity : " + dRes);
}
private void CompareFilesBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_CompareFilesBtnActionPerformed
Thread t = new Thread() {
@Override
public void run() {
DoCompareFiles();
unregisterThread(this);
}
};
registerThread(t);
t.start();
}//GEN-LAST:event_CompareFilesBtnActionPerformed
private void SelectSecondTestFileBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_SelectSecondTestFileBtnActionPerformed
// Select a dir
JFileChooser fc = new JFileChooser();
fc.setCurrentDirectory((FilePathEdt.getText().length() == 0) ? new java.io.File(".") : new java.io.File(FilePathEdt.getText()));
fc.setSelectedFile(fc.getCurrentDirectory());
fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
int iRet = fc.showOpenDialog(this);
if (iRet == JFileChooser.APPROVE_OPTION)
SelectSecondInputFileEdt.setText(fc.getSelectedFile().getAbsolutePath());
}//GEN-LAST:event_SelectSecondTestFileBtnActionPerformed
private void jButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton1ActionPerformed
String sStr = TermEdt.getText();
getSymbolsByEntropy(sStr, 3);
// Sample String:
// Is this, actual, text a test in deed? Yes! It should be...
// Check if current point belongs to a ''normal'' sequence of symbols
// <= This would only mean that we
// have met such a sequence in the training set. What did I just say? Who cares...
appendToLog(gr.demokritos.iit.jinsect.utils.printList(Arrays.asList(chunkString(sStr))));
//chunkString(sStr,2);
//chunkString(sStr,3);
}//GEN-LAST:event_jButton1ActionPerformed
private void DoBreakFileDown() {
String sDataString = "";
try {
ByteArrayOutputStream bsOut = new ByteArrayOutputStream();
FileInputStream fiIn = new FileInputStream(SelectInputFileEdt.getText());
int iData = 0;
while ((iData = fiIn.read()) > -1)
bsOut.write(iData);
sDataString = bsOut.toString();
}
catch (IOException ioe) {
ioe.printStackTrace();
}
//String [] saSubtexts = chunkString(sDataString);
List saSubtexts = Chunker.chunkString(sDataString);
// Show status window
final gr.demokritos.iit.jinsect.gui.StatusFrame fStatus =
new gr.demokritos.iit.jinsect.gui.StatusFrame();
fStatus.setVisible(true);
int iCnt = 0;
try {
//Iterator iStrIter = Arrays.asList(saSubtexts).iterator();
Iterator iStrIter = saSubtexts.iterator();
while (iStrIter.hasNext())
{
String sStr = (String)iStrIter.next();
fStatus.setStatus("Extracting meanings...", (double)(++iCnt) /
saSubtexts.size());
// DEBUG LINES
// appendToLog("\nCHECKING\n" + sStr + "\n");
//////////////
// Analyse chunk in substrings
List lSubStrings = utils.getSubStrings(sStr, sStr.length(), this);
if (lSubStrings.size() == 0)
continue;
// Keep most promising substrings, i.e. with the minimal parts
// lSubStrings = KeepLeastSizeSets(lSubStrings, 2);
appendToLog(utils.printList(lSubStrings));
ArrayList lOptions = new ArrayList();
lOptions.addAll(lSubStrings);
Iterator iIter = lOptions.iterator();
HashMap hSubstringSet = new HashMap();
while (iIter.hasNext()) {
Object oNext = iIter.next();
List lNext;
if (oNext instanceof List) {
lNext = (List)oNext;
}
else
{
lNext = new ArrayList();
lNext.add(oNext);
}
if (hSubstringSet.containsKey(lNext.toString()))
continue;
appendToLog("Case " + utils.printList(lNext));
hSubstringSet.put(lNext.toString(),1);
// Create vertex list
List lNodes = new ArrayList();
Iterator iSubstrings = lNext.iterator();
while (iSubstrings.hasNext()) {
lNodes.add(new VertexImpl(iSubstrings.next()));
}
// Attempt retrieval of union meanings
Iterator iNodes = lNodes.iterator();
String sUnionMeaning = "";
while (iNodes.hasNext()) {
String sCur = ((Vertex)iNodes.next()).toString();
Object oTxt = siIndex.getMeaning(new VertexImpl(sCur));
if (oTxt != null)
sUnionMeaning += "-" + siIndex.meaningToString(oTxt) + "-";
else
appendToLog("No meaning found...");
}
// if (sUnionMeaning.length() > 0)
// {
// appendToLog("MEANING: \n" + sUnionMeaning + "\nEND MEANING\n");
//
// // int iAnswer = JOptionPane.showConfirmDialog(this, "Do you want to look up constituent nodes?");
// // if (iAnswer == JOptionPane.NO_OPTION)
// // continue;
// // if (iAnswer == JOptionPane.CANCEL_OPTION)
// // return;
// continue;
// }
// else
// continue;
// Search common subnode
// Vertex vRes = sgOverallGraph.getCommonSubnode(lNodes);
//
// boolean bFound;
// if (vRes != null)
// {
// appendToLog(vRes.toString());
// Object oTxt = siIndex.getMeaning(vRes);
// if (oTxt != null)
// appendToLog(oTxt.toString());
// else
// appendToLog("No meaning found...");
// }
// else
// appendToLog("No common child node found... Backtracking...");
//
// bFound = vRes != null;
// int iSize = lNodes.size();
//
// while (!bFound) {
// // While not a single node list
// while (--iSize > 1) {
// Union uPossible = jinsect.utils.getCombinationsBy(lNodes, iSize);
// Iterator iPossibleIter = uPossible.iterator();
// while (iPossibleIter.hasNext()) {
// Concatenation cCur = (Concatenation)iPossibleIter.next();
// vRes = sgOverallGraph.getCommonSubnode(cCur);
// if (vRes != null) {
// appendToLog(vRes.toString());
// bFound = true;
// Object oTxt = siIndex.getMeaning(vRes);
// if (oTxt != null)
// appendToLog(oTxt.toString());
// }
// }
// if (bFound) // Found children. Exit loop.
// break;
// }
// if (iSize <= 1)
// break;
// }
// if (!bFound)
// appendToLog("No common child node found...");
}
}
}
finally
{
fStatus.setVisible(false);
}
}
private void BreakFileDownBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_BreakFileDownBtnActionPerformed
Thread t = new Thread() {
@Override
public void run() {
DoBreakFileDown();
unregisterThread(this);
}
};
registerThread(t);
t.start();
}//GEN-LAST:event_BreakFileDownBtnActionPerformed
private void SelectTestFileBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_SelectTestFileBtnActionPerformed
// Select a dir
JFileChooser fc = new JFileChooser();
fc.setCurrentDirectory((FilePathEdt.getText().length() == 0) ? new java.io.File(".") : new java.io.File(FilePathEdt.getText()));
fc.setSelectedFile(fc.getCurrentDirectory());
fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
int iRet = fc.showOpenDialog(this);
if (iRet == JFileChooser.APPROVE_OPTION)
SelectInputFileEdt.setText(fc.getSelectedFile().getAbsolutePath());
}//GEN-LAST:event_SelectTestFileBtnActionPerformed
private void ClearGraphsBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ClearGraphsBtnActionPerformed
// Init Graph
int Levels = Math.max(NGramSizeSldr.getValue(), 2);
sgOverallGraph = new SymbolicGraph(1, Levels); // Init graph with a min of 2
// Init Docs
Iterator iIter = Arrays.asList(cdDoc).iterator();
while (iIter.hasNext()) {
DistributionDocument d = (DistributionDocument)iIter.next();
d.clearDocumentGraph();
}
// Init index
siIndex = new SemanticIndex(sgOverallGraph);
try {
siIndex.MeaningExtractor = new LocalWordNetMeaningExtractor();
}
catch (IOException ioe) {
siIndex.MeaningExtractor = null; // Use default
}
}//GEN-LAST:event_ClearGraphsBtnActionPerformed
private void FindPathBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_FindPathBtnActionPerformed
// TODO: Implement correctly
// TEST - OK
/*
SymbolicGraph g = new SymbolicGraph(1, 9);
try {
g.addEdge(new VertexImpl("a"), new VertexImpl("b"));
g.addEdge(new VertexImpl("a"), new VertexImpl("c"));
g.addEdge(new VertexImpl("b"), new VertexImpl("d"));
g.addEdge(new VertexImpl("b"), new VertexImpl("e"));
g.addEdge(new VertexImpl("e"), new VertexImpl("g"));
g.addEdge(new VertexImpl("f"), new VertexImpl("d"));
g.addEdge(new VertexImpl("f"), new VertexImpl("g"));
g.addEdge(new VertexImpl("g"), new VertexImpl("f"));
g.addEdge(new VertexImpl("f"), new VertexImpl("a"));
List lRes = g.getPathBetween(new VertexImpl("f"), new VertexImpl("f"));
if (lRes != null)
appendToLog(lRes.toString());
else
appendToLog("Not found...");
ArrayList alNodes = new ArrayList();
alNodes.add(new VertexImpl("a"));
alNodes.add(new VertexImpl("f"));
Vertex vRes = g.getCommonSubnode(alNodes);
if (vRes != null)
appendToLog(vRes.toString());
else
appendToLog("No common child node found...");
}
catch (Exception e) {
e.printStackTrace();
}
*/
// TODO: Check for other interpretations
SymbolicGraph g = new SymbolicGraph(1, 9);
g.setDataString("abcdefg");
ArrayList alNodes = new ArrayList();
alNodes.add(new VertexImpl("c"));
alNodes.add(new VertexImpl("e"));
alNodes.add(new VertexImpl("h"));
// Vertex vRes = g.getCommonSubnode(alNodes);
// if (vRes != null)
// appendToLog(vRes.toString());
// else
// appendToLog("No common child node found...");
//
// Search common subnode
Vertex vRes = g.getCommonSubnode(alNodes);
boolean bFound;
if (vRes != null)
appendToLog(vRes.toString());
else
appendToLog("No common child node found... Backtracking...");
bFound = vRes != null;
int iSize = alNodes.size();
while (!bFound) {
// While not a single node list
while (--iSize > 1) {
Union uPossible = gr.demokritos.iit.jinsect.utils.getCombinationsBy(alNodes, iSize);
Iterator iPossibleIter = uPossible.iterator();
while (iPossibleIter.hasNext()) {
Concatenation cCur = (Concatenation)iPossibleIter.next();
vRes = g.getCommonSubnode(cCur);
if (vRes != null) {
appendToLog(vRes.toString());
bFound = true;
}
}
if (bFound) // Found children. Exit loop.
break;
}
}
if (!bFound)
appendToLog("No common child node found...");
}//GEN-LAST:event_FindPathBtnActionPerformed
public final boolean match(Object o) {
String sStr = (String)o;
if (sStr.length() < 2)
return false; // TODO: Check if true or false
int iCnt = Math.min(cdDoc.length, sStr.length()) - 1;
// Search for a full match
if (iCnt < sStr.length() - 1) // Not a full match
return false;
double dNorm = cdDoc[iCnt].normality(sStr);
// DEBUG LINES
// System.out.println("Matched " + o.toString() + " for a normality of " + dNorm + ".");
//////////////
return dNorm >= 0.5;
}
private void BreakDownBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_BreakDownBtnActionPerformed
String sStr = TermEdt.getText();
List lSubStrings = utils.getSubStrings(sStr, sStr.length(), this);
appendToLog(utils.printList(lSubStrings));
// Iterator iSubStrings = lSubStrings.iterator();
ArrayList lOptions = new ArrayList();
lOptions.addAll(lSubStrings);
/*
while (iSubStrings.hasNext()) {
lOptions.add(jinsect.utils.getFlattenedList(iSubStrings.next()));
}
*/
Iterator iIter = lOptions.iterator();
HashMap hSubstringSet = new HashMap();
while (iIter.hasNext()) {
List lNext = (List)iIter.next();
if (hSubstringSet.containsKey(lNext.toString()))
continue;
appendToLog("Case " + utils.printList(lNext));
hSubstringSet.put(lNext.toString(),1);
// Create vertex list
List lNodes = new ArrayList();
Iterator iSubstrings = lNext.iterator();
while (iSubstrings.hasNext()) {
lNodes.add(new VertexImpl(iSubstrings.next()));
}
int iAnswer = JOptionPane.showConfirmDialog(this, "Do you want to look up constituent nodes?");
if (iAnswer == JOptionPane.NO_OPTION)
continue;
if (iAnswer == JOptionPane.CANCEL_OPTION)
return;
// Search common subnode
Vertex vRes = sgOverallGraph.getCommonSubnode(lNodes);
boolean bFound;
if (vRes != null)
{
appendToLog(vRes.toString());
Object oTxt = siIndex.getMeaning(vRes);
appendToLog(siIndex.meaningToString(oTxt));
}
else
appendToLog("No common child node found... Backtracking...");
bFound = vRes != null;
int iSize = lNodes.size();
while (!bFound) {
// While not a single node list
while (--iSize > 1) {
Union uPossible = gr.demokritos.iit.jinsect.utils.getCombinationsBy(lNodes, iSize);
Iterator iPossibleIter = uPossible.iterator();
while (iPossibleIter.hasNext()) {
Concatenation cCur = (Concatenation)iPossibleIter.next();
vRes = sgOverallGraph.getCommonSubnode(cCur);
if (vRes != null) {
appendToLog(vRes.toString());
bFound = true;
Object oTxt = siIndex.getMeaning(vRes);
appendToLog(siIndex.meaningToString(oTxt));
}
}
if (bFound) // Found children. Exit loop.
break;
}
if (iSize <= 1)
break;
}
if (!bFound)
appendToLog("No common child node found...");
}
}//GEN-LAST:event_BreakDownBtnActionPerformed
private void LookUpTermBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_LookUpTermBtnActionPerformed
// For every level of n-grams in descending order
String sStr = TermEdt.getText();
appendToLog("Looking up term:" + sStr);
appendToLog("===============");
for (int iCnt=0; iCnt < Math.min(cdDoc.length, sStr.length()); iCnt++)
// Display distribution similarity
appendToLog("Level " + String.valueOf(iCnt) + ":" +
String.valueOf(cdDoc[iCnt].normality(sStr)));
appendToLog("===DONE========\n");
}//GEN-LAST:event_LookUpTermBtnActionPerformed
private void appendToLog(final String s) {
Runnable rQuick = new Runnable() {
public void run() {
LogMemo.append(s + "\n");
System.err.println(s);
}
};
// Do not update image more often that 1 / sec.
// if ((new Date()).getTime() - dLastUpdate.getTime() > 1000)
// {
dLastUpdate = new Date();
try {
SwingUtilities.invokeLater(rQuick);
}
catch (Exception e) {
// Ignore
}
// };
}
/** Extract language symbols from candidate text, based on entropy
* calculation.
*
* @param sStr The string from which symbols are extracted.
* @param iMaxSymbolSize The maximum size of extracted symbols.
* @return A {@link List} indicating the extracted symbols.
*/
private List getSymbolsByEntropy(String sStr, int iMaxSymbolSize) {
ArrayList alRes = new ArrayList();
// For each n-gram size find minimum entropy n-grams
String sSubStr;
TreeMap tmRes = new TreeMap();
// For each n-gram size
for (int iNGramSize = 1; iNGramSize < iMaxSymbolSize; iNGramSize++) {
// For each n-gram of selected size
for (int iCnt = 0; iCnt <= sStr.length() - iNGramSize; iCnt++) {
if (iCnt + iNGramSize > sStr.length())
continue;
// Get n-gram
sSubStr = sStr.substring(iCnt, iCnt + iNGramSize);
// Look-up current n-gram
Vertex vStrNode = gr.demokritos.iit.jinsect.utils.locateVertexInGraph(sgOverallGraph, sSubStr);
// Ignore inexistent symbols
if (vStrNode == null)
continue;
// else get outgoing edges
List lEdges = gr.demokritos.iit.jinsect.utils.getOutgoingEdges(sgOverallGraph, vStrNode);
Iterator iEdgeIter = lEdges.iterator();
// Get children distribution
Distribution dDist = new Distribution();
if (lEdges.size() > 0) {
while (iEdgeIter.hasNext()) {
WeightedEdge weCur = (WeightedEdge)iEdgeIter.next();
dDist.setValue(weCur.toString(), weCur.getWeight());
}
// Determine entropy of children distribution
dDist.normalizeToSum();
double dEntropy = statisticalCalculation.entropy(dDist);
tmRes.put(dEntropy, sSubStr);
// DEBUG LINES
// appendToLog(vStrNode.toString() + " entropy: " + dEntropy);
//////////////
}
}
// DEBUG LINES
// Object oMin = tmRes.firstKey();
// Formatter f = new Formatter();
// appendToLog("'" + tmRes.get(oMin).toString() + "' has minimum entropy of " +
// f.format("%6.3f", oMin));
//
// Iterator iKeys = tmRes.keySet().iterator();
// String sOrderedList = "";
// while (iKeys.hasNext()) {
// Object oKey = iKeys.next();
// sOrderedList += "'" + tmRes.get(oKey).toString() + "' (" + oKey + " => " + Math.pow((Double)oKey, 2) + " alternatives)\n";
// }
// sOrderedList += " is the ordering of possible symbols.";
// appendToLog(sOrderedList);
//////////////
}
// Select most promising ones
// Add to symbol pool
return alRes;
}
// Chunks string according to maximum entropy of 1-grams
private String[] chunkString(String sStr)
{
return chunkString(sStr, 1);
}
private TreeMap identifyCandidateDelimiters(String sStr, int iNGramSize) {
String sSubStr;
Integer[] iRes;
ArrayList alRes = new ArrayList();
TreeMap tmRes = new TreeMap();
for (int iCnt = 0; iCnt <= sStr.length() - iNGramSize; iCnt++) {
if (iCnt + iNGramSize > sStr.length())
continue;
// Get n-gram
sSubStr = sStr.substring(iCnt, iCnt + iNGramSize);
if (tmRes.containsValue(sSubStr)) // Ignore duplicates
continue;
// Look-up current n-gram
Vertex vStrNode = gr.demokritos.iit.jinsect.utils.locateVertexInGraph(sgOverallGraph, sSubStr);
if (vStrNode == null)
continue; // Ignore inexistent symbols
// double dNormEntropy = getEntropyOfNextChar(sSubStr, true);
double dEntropy = getEntropyOfNextChar(sSubStr, false);
// tmRes.put(dNormEntropy, sSubStr);
tmRes.put(dEntropy, sSubStr);
}
return tmRes;
}
private double getEntropyOfNextChar(String sStr) {
return getEntropyOfNextChar(sStr, false);
}
private final double getEntropyOfNextChar(String sStr, boolean bNormalized) {
double dRes = 0.0;
// Look-up current n-gram
Vertex vStrNode = gr.demokritos.iit.jinsect.utils.locateVertexInGraph(sgOverallGraph, sStr);
if (vStrNode == null)
return dRes; // Ignore inexistent symbols
// else get outgoing edges
List lEdges = gr.demokritos.iit.jinsect.utils.getOutgoingEdges(sgOverallGraph, vStrNode);
Iterator iEdgeIter = lEdges.iterator();
Distribution dDist = new Distribution();
if (lEdges.size() > 0) {
while (iEdgeIter.hasNext()) {
WeightedEdge weCur = (WeightedEdge)iEdgeIter.next();
if ( Double.isNaN(weCur.getWeight()))
System.err.println("WARNING: Not a number edge weight for edge:" + weCur.toString());
dDist.setValue(weCur.toString(), weCur.getWeight());
}
dDist.normalizeToSum();
if (bNormalized) {
// Calc NORMALIZED entropy - entropy to the number of appearences
double dLogOccurences = (Math.log(dDist.calcTotalValues()) / Math.log(2));
dRes = statisticalCalculation.entropy(dDist) / dLogOccurences;
}
else
// Calc entropy
dRes = statisticalCalculation.entropy(dDist);
}
if ( Double.isNaN(dRes))
System.err.println("WARNING: Not a number entropy for symbol:" + vStrNode);
return dRes;
}
/** Returns the probability of generation of an exact given string, based on the alphabet of the
* Symbolic Graph, and given that there the generation process is random.
*@param sString The string for which the probability of appearence is to be calculated.
*@return The probability of appearence of the given string.
*/
private final double getProbabilityOfStringInRandomText(String sPrefix, String sSuffix) {
double dRes = 0.0;
// Look directly in the data string of the symbolic graph
int iPrefixCount = 0;
int iLastOccurence = -1;
if (sPrefix.length() == 0) // Check for empty string
return 1.0;
while ((iLastOccurence = sgOverallGraph.getDataString().indexOf(sPrefix, iLastOccurence + 1))
> -1)
iPrefixCount++;
// Consider that the occurence of a random suffix is a proportion of the prefix occurences,
// as indicated by the probability of the given suffix, as a random selection process of n
// symbols from the symbol set.
int iFullStringCount = (int)Math.ceil((double)iPrefixCount *
Math.pow(1.0 / sgOverallGraph.getAlphabet().size(), sSuffix.length()));
// Consider p(sPrefix) = N(sPrefix) / (length(Text) / length(sPrefix))
double pPrefix = (double) iPrefixCount /
(sgOverallGraph.getDataString().length() / sPrefix.length());
String sFullString = sPrefix + sSuffix;
double pJoined = (double) iFullStringCount /
(sgOverallGraph.getDataString().length() / sFullString.length());
// Return p(sPrefix) * p(sPrefix, sSuffix)
dRes = (iPrefixCount == 0) ? 0.0 : pPrefix * pJoined;
return dRes;
}
/** Returns the probability of occurence of a given suffix, given a prefix, within the data string
*of the Symbolic Graph.
*@param sPrefix The prefix required.
*@param sSuffix The suffix for which the probability of occurence is to be calculated, given the
* prefix.
*@return The probability of occurence of the suffix, given the prefix.
*/
private final double getProbabilityOfStringInText(String sPrefix, String sSuffix) {
double dRes = 0.0;
// Look directly in the data string of the symbolic graph
int iPrefixCount = 0;
int iLastOccurence = -1;
if (sPrefix.length() == 0) // Check for empty string
return 1.0;
while ((iLastOccurence = sgOverallGraph.getDataString().indexOf(sPrefix, iLastOccurence + 1))
> -1)
iPrefixCount++;
String sFullString = sPrefix + sSuffix;
int iFullStringCount = 0;
iLastOccurence = -1;
while ((iLastOccurence = sgOverallGraph.getDataString().indexOf(sFullString, iLastOccurence + 1))
> -1)
iFullStringCount++;
// Consider p(sPrefix) = N(sPrefix) / (length(Text) / length(sPrefix))
double pPrefix = (double) iPrefixCount /
(sgOverallGraph.getDataString().length() / sPrefix.length());
double pJoined = (double) iFullStringCount /
(sgOverallGraph.getDataString().length() / sFullString.length());
// Return p(sPrefix) * p(sPrefix, sSuffix)
dRes = (iPrefixCount == 0) ? 0.0 : pPrefix * pJoined;
return dRes;
}
// Returns a list of indices concerning possible split points
private Integer[] splitPointsByDelimiterList(String sStr, char[] lDelimiters) {
TreeMap tmDels = new TreeMap();
for (int iCnt=0; iCnt < lDelimiters.length; iCnt++)
tmDels.put(iCnt, new String() + lDelimiters[iCnt]);
return splitPointsByDelimiterList(sStr, tmDels);
}
// Returns a list of indices concerning possible split points
private Integer[] splitPointsByDelimiterList(String sStr, SortedMap lDelimiters) {
ArrayList alRes = new ArrayList();
TreeMap lLocal = new TreeMap();
lLocal.putAll(lDelimiters);
// For every candidate delimiter
while (lLocal.size() > 0) {
Object oNext = lLocal.lastKey();
// Get all split points
int iNextSplit = 0;
int iLastSplit = 0;
while ((iNextSplit = sStr.indexOf((String)lDelimiters.get(oNext), iLastSplit)) > -1)
{
// TODO : Check
alRes.add(new Integer(iNextSplit + ((String)lDelimiters.get(oNext)).length()));
iLastSplit = iNextSplit + 1;
}
lLocal.remove(oNext);
}
Integer [] iaRes = new Integer[alRes.size()];
alRes.toArray(iaRes);
gr.demokritos.iit.jinsect.utils.bubbleSortArray(iaRes);
return iaRes;
}
/* Chunks string according to maximum entropy of (iNGramSize)-grams
*@param sStr The string to chunk.
*@param iNGramSize The n-gram size to take into account for delimiter analysis.
*@return An array of substrings, which are chunks of the original string.
*/
private String[] chunkString(String sStr, int iNGramSize) {
// Get separation points
Integer[] iRes = splitPointsByDelimiterList(sStr, getDelimiters());
String[] sRes = splitStringByDelimiterPoints(sStr, iRes);
// DEBUG LINES
appendToLog("Text splitted into "+ sRes.length + " chunks.");
// appendToLog(jinsect.utils.printList(Arrays.asList(sRes), "\n*"));
//appendToLog("*************");
//////////////
return sRes;
}
private double DetermineWMeanMutualInformationOf(String sStr, int iPos,
int iMaxDistance, int iMinDistance, boolean bBefore) {
double dMeanMutualInfo = 0.0;
for (int iCurDist = iMinDistance; iCurDist < iMaxDistance; iCurDist++) {
// Should check if the delimiter is part of the adjacent sequence of symbols
// Using pointwise mutual information
int iSubStrStart = Math.max(iPos - iCurDist, 0);
int iSubStrEnd = Math.min(iPos + iCurDist, sStr.length());
double dPrvEntropy, dCurEntropy;
if (bBefore) {
// Get uncertainty of next symbol, given the preceding substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos));
// Get uncertainty of next symbol, given the preceding substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos + 1));
// Small decrease or increase in the entropy, shows that the candidate is not very important to the decision of the next
// symbol and thus, is probably part of an existing word.
}
else {
// Get uncertainty of next symbol, given the following substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iPos + 1, iSubStrEnd));
// Get uncertainty of next symbol, given the following substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iPos, iSubStrEnd));
}
double dMutualInformationBefore = dCurEntropy - dPrvEntropy;
// Closest tests should have maximum impact
dMeanMutualInfo += (iMaxDistance - iCurDist + 1) * dMutualInformationBefore;
}
// Calc sum of integers in range [iMinDistance,iMaxDistance]
int iSum = 0;
for (int iCnt = iMinDistance; iCnt < iMaxDistance; iCnt++)
iSum += iCnt;
dMeanMutualInfo /= iSum; // Get weighted mean
return dMeanMutualInfo;
}
private double DetermineMeanMutualInformationOf(String sStr, int iPos,
int iMaxDistance, int iMinDistance, boolean bBefore) {
double dMeanMutualInfo = 0.0;
for (int iCurDist = iMinDistance; iCurDist < iMaxDistance; iCurDist++) {
// Should check if the delimiter is part of the adjacent sequence of symbols
// Using pointwise mutual information
int iSubStrStart = Math.max(iPos - iCurDist, 0);
int iSubStrEnd = Math.min(iPos + iCurDist, sStr.length());
double dPrvEntropy, dCurEntropy;
if (bBefore) {
// Get uncertainty of next symbol, given the preceding substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos));
// Get uncertainty of next symbol, given the preceding substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos + 1));
// Small change in the entropy, shows that the candidate is not very important to the decision of the next
// symbol and thus, is probably part of an existing word.
}
else {
// Get uncertainty of next symbol, given the following substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iPos + 1, iSubStrEnd));
// Get uncertainty of next symbol, given the following substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iPos, iSubStrEnd));
}
double dMutualInformationBefore = dCurEntropy - dPrvEntropy;
dMeanMutualInfo += dMutualInformationBefore;
}
dMeanMutualInfo /= iMaxDistance - iMinDistance; // Get mean
return dMeanMutualInfo;
}
private Double[] DetermineMutualInformationDistributionOf(String sStr,
int iPos, int iMaxDistance, int iMinDistance, boolean bBefore) {
Distribution dRes = new Distribution();
double dMeanMutualInfo = 0.0;
for (int iCurDist = iMinDistance; iCurDist < iMaxDistance; iCurDist++) {
// Should check if the delimiter is part of the adjacent sequence of symbols
// Using pointwise mutual information
int iSubStrStart = Math.max(iPos - iCurDist, 0);
int iSubStrEnd = Math.min(iPos + iCurDist, sStr.length());
double dPrvEntropy, dCurEntropy;
if (bBefore) {
// Get uncertainty of next symbol, given the preceding substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos));
// Get uncertainty of next symbol, given the preceding substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iSubStrStart, iPos + 1));
// Small change in the entropy, shows that the candidate is not very important to the decision of the next
// symbol and thus, is probably part of an existing word.
}
else {
// Get uncertainty of next symbol, given the following substring
dPrvEntropy = getEntropyOfNextChar(sStr.substring(iPos + 1, iSubStrEnd));
// Get uncertainty of next symbol, given the following substring PLUS the delimiter
dCurEntropy = getEntropyOfNextChar(sStr.substring(iPos, iSubStrEnd));
}
double dMutualInformationBefore = dCurEntropy - dPrvEntropy;
dRes.setValue(iCurDist, dMutualInformationBefore);
}
Double[] dblRes = new Double[dRes.asTreeMap().size()];
dRes.asTreeMap().values().toArray(dblRes);
return dblRes;
}
private Integer [] evalAndSelectActualDelimiterPoints(Integer[] iaDelimPoints, String sStr) {
int iCurDelim = 0;
ArrayList alList = new ArrayList();
// DEBUG LINES
try {
FileWriter fOut = new FileWriter("train.arff");
HashMap hDelims = new HashMap();
for (int iCnt = 0; iCnt < iaDelimPoints.length; iCnt++) {
String sChar = sStr.substring(iaDelimPoints[iCnt], Math.min(iaDelimPoints[iCnt] + 1, sStr.length()));
hDelims.put(String.valueOf(sChar.hashCode()), sChar);
}
fOut.write("@relation train"+ System.getProperty("line.separator")
+ System.getProperty("line.separator"));
// Write delimiters
fOut.write("@attribute delimChar {");
String sLegend = "";
Iterator iIter = hDelims.keySet().iterator();
while (iIter.hasNext()) {
String sNext = (String)iIter.next();
fOut.write(sNext);
if (Character.isISOControl(sNext.charAt(0)) ||
Character.isWhitespace(sNext.charAt(0)))
sLegend += "% " + sNext + ": chr(" + sNext + ")" + System.getProperty("line.separator");
else
sLegend += "% " + sNext + ": chr(" + hDelims.get(sNext) + ")" + System.getProperty("line.separator");
if (iIter.hasNext())
fOut.write(",");
}
fOut.write("}" + System.getProperty("line.separator"));
//////////////////
fOut.write("@attribute class {delim,nondelim}" + System.getProperty("line.separator"));
for (int iCnt=0; iCnt < NGramSizeSldr.getValue(); iCnt++) {
fOut.write("@attribute entropyBefore" + iCnt + " real" + System.getProperty("line.separator"));
}
for (int iCnt=0; iCnt < NGramSizeSldr.getValue(); iCnt++) {
fOut.write("@attribute entropyAfter" + iCnt + " real" + System.getProperty("line.separator"));
}
fOut.write(System.getProperty("line.separator") + System.getProperty("line.separator"));
fOut.write("@data" + System.getProperty("line.separator"));
// For all delimiter points
while (iCurDelim < iaDelimPoints.length) {
// DO NOT USE
// Check if current point belongs to a ''normal'' sequence of symbols <= This would only mean that we
// have met such a sequence in the training set.
// int iSubStrStart = Math.max(iaDelimPoints[iCurDelim] - (NGramSizeSldr.getValue() / 2), 0);
// int iSubStrEnd = Math.min(iaDelimPoints[iCurDelim] + (NGramSizeSldr.getValue() / 2), sStr.length());
// If not in a normal pattern.
// if (!match(sStr.substring(iSubStrStart, iSubStrEnd)))
// alList.add(iaDelimPoints[iCurDelim]);
// Should check if the delimiter is part of the adjacent sequence of symbols
// Using pointwise mutual information
double dMutualInformationBefore = DetermineMeanMutualInformationOf(sStr, iaDelimPoints[iCurDelim],
Math.max(NGramSizeSldr.getValue(), 1), 1, true);
double dMutualInformationAfter = DetermineMeanMutualInformationOf(sStr, iaDelimPoints[iCurDelim],
Math.max(NGramSizeSldr.getValue(), 1), 1, false);
int iSubStrStart = Math.max(iaDelimPoints[iCurDelim] - NGramSizeSldr.getValue(), 0);
int iSubStrEnd = Math.min(iaDelimPoints[iCurDelim] + NGramSizeSldr.getValue(), sStr.length());
// appendToLog("Symbol '" + sStr.substring(iaDelimPoints[iCurDelim], Math.min(iaDelimPoints[iCurDelim] + 1, sStr.length())) +
// "' in '" + sStr.substring(iSubStrStart, Math.min(iSubStrEnd + 1, sStr.length())) + "' has mutual information pair of (" +
// String.format("%6.4f", dMutualInformationBefore) + "," +
// String.format("%6.4f", dMutualInformationAfter) + ")\n");
//////////////
// Output category
fOut.write(sStr.substring(iaDelimPoints[iCurDelim],
Math.min(iaDelimPoints[iCurDelim] + 1, sStr.length())).hashCode() + ",");
if (" ()-\n".indexOf(sStr.substring(iaDelimPoints[iCurDelim], Math.min(iaDelimPoints[iCurDelim] + 1, sStr.length()))) > -1)
fOut.write("delim,");
else
fOut.write("nondelim,");
// Output features
Double[] dBefore =
DetermineMutualInformationDistributionOf(sStr, iaDelimPoints[iCurDelim],
Math.max(NGramSizeSldr.getValue(), 1), 1, true);
Double[] dAfter = DetermineMutualInformationDistributionOf(sStr, iaDelimPoints[iCurDelim],
Math.max(NGramSizeSldr.getValue(), 1), 1, false);
String sMutualInformationBefore = "";
// Pad with zeroes
if (dBefore.length < NGramSizeSldr.getValue())
for (int iCnt = dBefore.length; iCnt < NGramSizeSldr.getValue(); iCnt++)
sMutualInformationBefore += "0.0,";
for (int iCnt = 0; iCnt < dBefore.length; iCnt++) {
sMutualInformationBefore += dBefore[iCnt];
if (iCnt < dBefore.length - 1)
sMutualInformationBefore += ",";
}
String sMutualInformationAfter = "";
// Pad with zeroes
if (dAfter.length < NGramSizeSldr.getValue())
for (int iCnt = dAfter.length; iCnt < NGramSizeSldr.getValue(); iCnt++)
sMutualInformationAfter += "0.0,";
for (int iCnt = 0; iCnt < dAfter.length; iCnt++) {
sMutualInformationAfter += dAfter[iCnt];
if (iCnt < dAfter.length - 1)
sMutualInformationAfter += ",";
}
fOut.write(sMutualInformationBefore + "," +
sMutualInformationAfter + System.getProperty("line.separator"));
fOut.flush();
if ((dMutualInformationBefore > 1) && (dMutualInformationAfter < 1)) {
// DEBUG LINES
// appendToLog("DELIMITER");
//////////////
alList.add(iaDelimPoints[iCurDelim]);
}
++iCurDelim;
}
fOut.close();
}
catch (IOException ioe) {
System.out.println("Output failed.");
}
Integer[] iaRes = new Integer[alList.size()];
iaRes = (Integer[])alList.toArray(iaRes);
return iaRes;
}
private int determineImportantDelimiters(SortedMap smMap) {
Iterator iIter = smMap.keySet().iterator();
// Distribution dEntropyDist = new Distribution();
// Distribution dEntropyDeltaDist = new Distribution();
Distribution dDist = new Distribution();
Distribution dReverse = new Distribution();
// Get first number
Double dPrv = Double.NEGATIVE_INFINITY;
Double dTwoPrv = Double.NEGATIVE_INFINITY;
// Create corresponding distribution
while (iIter.hasNext()) {
Double oNext = (Double)iIter.next();
if ((dPrv != Double.NEGATIVE_INFINITY) && (dTwoPrv != Double.NEGATIVE_INFINITY)) {
if (oNext.isNaN())
System.err.println("WARNING: Encountered NaN. Ignoring...");
// dEntropyDeltaDist.asTreeMap().put(smMap.get(oNext), (oNext - dPrv)); // Get distance from previous data point
// dEntropyDist.asTreeMap().put(smMap.get(oNext), oNext); // Get position of current data point
dDist.setValue(dPrv, dPrv * Math.abs(dPrv-dTwoPrv-oNext+dPrv)); // Detect peaks
dReverse.setValue(dPrv * Math.abs(dPrv-dTwoPrv-oNext+dPrv), dPrv);
}
dTwoPrv = dPrv;
dPrv = oNext;
}
// DEBUG LINES
// System.err.println("Symbol\tEntropy");
// for (Iterator iEntropies = smMap.keySet().iterator();
// iEntropies.hasNext();) {
// Object o = iEntropies.next();
// String sSymbol = (String)smMap.get(o);
// try {
// sSymbol = URLEncoder.encode(sSymbol, "utf8");
// } catch (UnsupportedEncodingException ex) {
// sSymbol = "(NotPrintable)";
// ex.printStackTrace(System.err);
// }
// System.err.println(o.toString() + "\t" + sSymbol);
// }
//////////////
double dVar = dDist.variance(true);
double dMean = dDist.average(true);
// return getDelimiterIndexByThreshold(smMap, Math.min(dMean + Math.abs(dVar), dDist.maxValue()));
return getDelimiterIndexByThreshold(smMap, dReverse.getValue(dDist.maxValue()));
}
private int getDelimiterIndexByThreshold(SortedMap smMap, double dThreshold) {
// Locate delim in map
Iterator iIter = smMap.keySet().iterator();
int iCnt = 0;
while (iIter.hasNext()) {
if ((Double)iIter.next() > dThreshold)
break;
iCnt++;
}
// Indicate index
return smMap.size() - iCnt + 1;
}
// Determine corpus complexity
private void DoAnalyseCorpus() {
gr.demokritos.iit.jinsect.gui.StatusFrame fStatus = new gr.demokritos.iit.jinsect.gui.StatusFrame();
analyseCorpus(fStatus);
fStatus.dispose();
}
private double determineDistanceDeviation() {
double dRes = 0.0;
SortedMap smDelims = identifyCandidateDelimiters(sgOverallGraph.getDataString(), 1);
int iImportant = determineImportantDelimiters(smDelims);
Iterator iIter = smDelims.keySet().iterator();
int iCnt = 0;
while (iIter.hasNext() && (iCnt++ < smDelims.size() - iImportant))
iIter.next();
smDelims = smDelims.tailMap(iIter.next());
if (!smDelims.containsValue(StreamTokenizer.TT_EOF)) {
smDelims.put((Double)smDelims.lastKey() + 0.1, new StringBuffer().append((char)StreamTokenizer.TT_EOF).toString()); // Add EOF char
}
String[] saChunks = splitStringByDelimiterPoints(sgOverallGraph.getDataString(),
splitPointsByDelimiterList(sgOverallGraph.getDataString(), smDelims));
Distribution dSizes = new Distribution();
for (iCnt=0;iCnt < saChunks.length; iCnt++) {
dSizes.setValue((double)saChunks[iCnt].length(),
dSizes.getValue((double)saChunks[iCnt].length()) + 1);
}
dRes = dSizes.average(false) + dSizes.standardDeviation(false);
return dRes;
}
private void analyseCorpus(final IStatusDisplayer fStatus) {
// Constants
final String sDistrosFile = "Distros.tmp";
final String sSymbolsFile = "Symbols.tmp";
// Chunker re-init
Chunker = new EntropyChunker();
// Vars
int Levels = 2; // Analyze 2-grams
sgOverallGraph = new SymbolicGraph(1, Levels); // Init graph with a min of 2
siIndex = new SemanticIndex(sgOverallGraph);
try {
siIndex.MeaningExtractor = new LocalWordNetMeaningExtractor();
}
catch (IOException ioe) {
siIndex.MeaningExtractor = null; // Use default
}
try
{
// Get categories and files from disk
DocumentSet dsSet = new DocumentSet(FilePathEdt.getText(), 1.0);
dsSet.createSets(true, (double)100 / 100);
int iCurCnt, iTotal;
String sFile = "";
Iterator iIter = dsSet.getTrainingSet().iterator();
iTotal = dsSet.getTrainingSet().size();
if (iTotal == 0) // No documents to import
{
appendToLog("No input documents.\n");
appendToLog("======DONE=====\n");
return;
}
appendToLog("Training chunker...");
Chunker.train(dsSet.toFilenameSet(DocumentSet.FROM_WHOLE_SET));
appendToLog("Setting delimiters...");
setDelimiters(Chunker.getDelimiters());
iCurCnt = 0;
// Create overall document
cdDoc = new DistributionDocument[Levels];
for (int iCnt=0; iCnt < Levels; iCnt++)
cdDoc[iCnt] = new DistributionDocument(1, MinLevel + iCnt); // For all windows
fStatus.setVisible(true);
ThreadList t = new ThreadList(Runtime.getRuntime().availableProcessors() + 1);
appendToLog("(Pass 1/3) Loading files..." + sFile);
TreeSet tsOverallSymbols = new TreeSet();
while (iIter.hasNext())
{
sFile = ((CategorizedFileEntry)iIter.next()).getFileName();
//appendToLog("Loading file..." + sFile);
fStatus.setStatus("(Pass 1/3) Loading file..." + sFile, (double)iCurCnt / iTotal);
// Thread arguments
final DistributionDocument[] cdDocArg = cdDoc;
final String sFileArg = sFile;
for (int iCnt=0; iCnt < cdDoc.length; iCnt++) {
final int iCntArg = iCnt;
while (!t.addThreadFor(new Runnable() {
public void run() {
if (!RightToLeftText)
cdDocArg[iCntArg].loadDataStringFromFile(sFileArg,false);
else
{
// Use reverse
cdDocArg[iCntArg].setDataString(
utils.reverseString(utils.loadFileToString(sFileArg)),
iCntArg, false);
}
}
}))
Thread.yield();
}
try {
t.waitUntilCompletion();
} catch (InterruptedException ex) {
ex.printStackTrace(System.err);
appendToLog("Interrupted...");
sgOverallGraph.removeNotificationListener();
return;
}
// Construct the datastring by dividing text with an EOF char.
sgOverallGraph.setDataString(((new StringBuffer().append((char)StreamTokenizer.TT_EOF))).toString());
sgOverallGraph.loadFromFile(sFile);
// appendToLog("Loaded file..."+ sFile);
fStatus.setStatus("Loaded file..." + sFile, (double)++iCurCnt / iTotal);
Thread.yield();
}
// Get symbol set
Set sSymbols = null;
File fPreviousSymbols = new File(sSymbolsFile);
boolean bSymbolsLoadedOK = false;
if (fPreviousSymbols.exists()) {
System.err.println("ATTENTION: Using previous symbols...");
try {
FileInputStream fis = new FileInputStream(fPreviousSymbols);
ObjectInputStream ois = new ObjectInputStream(fis);
sSymbols = (Set)ois.readObject();
ois.close();
bSymbolsLoadedOK = true;
} catch (FileNotFoundException ex) {
ex.printStackTrace(System.err);
} catch (IOException ex) {
ex.printStackTrace(System.err);
// Reset distros
} catch (ClassNotFoundException ex) {
ex.printStackTrace(System.err);
}
}
if (!bSymbolsLoadedOK)
// If failed to load, extract...
sSymbols = getSymbolsByProbabilities(sgOverallGraph.getDataString(), fStatus);
int iMinSymbolSize = Integer.MAX_VALUE;
int iMaxSymbolSize = Integer.MIN_VALUE;
// Determine min, max symbol length, size distros
Iterator iSymbol = sSymbols.iterator();
// DEBUG LINES
// System.err.println("-----Symbol Start");
//////////////
while (iSymbol.hasNext()) {
String sCurSymbol = (String)iSymbol.next();
if (iMaxSymbolSize < sCurSymbol.length())
iMaxSymbolSize = sCurSymbol.length();
if (iMinSymbolSize > sCurSymbol.length())
iMinSymbolSize = sCurSymbol.length();
// DEBUG LINES
// System.err.println(sCurSymbol);
//////////////
}
// DEBUG LINES
// System.err.println("-----Symbol End");
//////////////
try {
// Save to file for future use
FileOutputStream fos = new FileOutputStream(sSymbolsFile);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(sSymbols);
oos.close();
} catch (FileNotFoundException ex) {
ex.printStackTrace(System.err);
} catch (IOException ex) {
ex.printStackTrace(System.err);
}
appendToLog("(Pass 2/3) Determining symbol distros per n-gram size...");
// Run through all files again to determine n-gram sizes
iIter = dsSet.getTrainingSet().iterator();
iTotal = dsSet.getTrainingSet().size();
if (iTotal == 0) // No documents to import
{
appendToLog("No input documents.\n");
appendToLog("======DONE=====\n");
return;
}
iCurCnt = 0;
Distribution dSymbolsPerSize = new Distribution();
Distribution dNonSymbolsPerSize = new Distribution();
Distribution dSymbolSizes = new Distribution();
// Use previous distros if available
File fPreviousRun = new File(sDistrosFile);
boolean bDistrosLoadedOK = false;
if (fPreviousRun.exists()) {
System.err.println("ATTENTION: Using previous distros...");
try {
FileInputStream fis = new FileInputStream(fPreviousRun);
ObjectInputStream ois = new ObjectInputStream(fis);
dSymbolsPerSize = (Distribution)ois.readObject();
dNonSymbolsPerSize = (Distribution)ois.readObject();
dSymbolSizes = (Distribution)ois.readObject();
ois.close();
bDistrosLoadedOK = true;
} catch (FileNotFoundException ex) {
ex.printStackTrace(System.err);
} catch (IOException ex) {
ex.printStackTrace(System.err);
// Reset distros
dSymbolsPerSize = new Distribution();
dNonSymbolsPerSize = new Distribution();
dSymbolSizes = new Distribution();
} catch (ClassNotFoundException ex) {
ex.printStackTrace(System.err);
// Reset distros
dSymbolsPerSize = new Distribution();
dNonSymbolsPerSize = new Distribution();
dSymbolSizes = new Distribution();
}
}
// If distros not loaded
if (!bDistrosLoadedOK)
while (iIter.hasNext())
{
// appendToLog("Parsing file..." + sFile);
fStatus.setStatus("(Pass 2/3) Parsing file..." + sFile, (double)iCurCnt++ / iTotal);
sFile = ((CategorizedFileEntry)iIter.next()).getFileName();
// Load file and determine max n-gram
String sDataString = "";
try {
ByteArrayOutputStream bsOut = new ByteArrayOutputStream();
FileInputStream fiIn = new FileInputStream(sFile);
int iData = 0;
while ((iData = fiIn.read()) > -1)
bsOut.write(iData);
sDataString = bsOut.toString();
}
catch (IOException ioe) {
ioe.printStackTrace(System.err);
}
// Thread args
final Distribution dSymbolsPerSizeArg = dSymbolsPerSize;
final Distribution dNonSymbolsPerSizeArg = dNonSymbolsPerSize;
final Distribution dSymbolSizesArg = dSymbolSizes;
final String sDataStringArg = sDataString;
final Set sSymbolsArg = sSymbols;
for (int iSymbolSize = iMinSymbolSize; iSymbolSize <= iMaxSymbolSize; iSymbolSize++) {
final int iSymbolSizeArg = iSymbolSize;
while (!t.addThreadFor(new Runnable() {
public void run() {
NGramDocument ndCur = new NGramDocument(iSymbolSizeArg, iSymbolSizeArg,
1, iSymbolSizeArg, iSymbolSizeArg);
ndCur.setDataString(sDataStringArg);
int iSymbolCnt = 0;
int iNonSymbolCnt = 0;
Iterator iExtracted =
ndCur.getDocumentGraph().getGraphLevel(0).getVertexSet().iterator();
while (iExtracted.hasNext()) {
String sCur = ((Vertex)iExtracted.next()).toString();
if (sSymbolsArg.contains(sCur)) {
iSymbolCnt++;
// Increase occurences of this symbol length
synchronized (dSymbolSizesArg) {
dSymbolSizesArg.setValue(sCur.length(),
dSymbolSizesArg.getValue(sCur.length()) + 1.0);
}
}
else
iNonSymbolCnt++;
}
// Update distributions
synchronized (dSymbolsPerSizeArg) {
dSymbolsPerSizeArg.setValue(iSymbolSizeArg,
dSymbolsPerSizeArg.getValue(iSymbolSizeArg) + iSymbolCnt);
}
synchronized (dNonSymbolsPerSizeArg) {
dNonSymbolsPerSizeArg.setValue(iSymbolSizeArg,
dNonSymbolsPerSizeArg.getValue(iSymbolSizeArg) + iNonSymbolCnt);
}
}
}))
Thread.yield();
}
}
if (!bDistrosLoadedOK)
try {
t.waitUntilCompletion();
try {
// Save to file for future use
FileOutputStream fos = new FileOutputStream(sDistrosFile);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(dSymbolsPerSize);
oos.writeObject(dNonSymbolsPerSize);
oos.writeObject(dSymbolSizes);
oos.close();
} catch (FileNotFoundException ex) {
ex.printStackTrace(System.err);
} catch (IOException ex) {
ex.printStackTrace(System.err);
}
//System.err.println("Symbols distro:" + dSymbolsPerSize.asTreeMap());
//System.err.println("Non-Symbols distro:" + dNonSymbolsPerSize.asTreeMap());
} catch (InterruptedException ex) {
appendToLog("Interrupted...");
sgOverallGraph.removeNotificationListener();
return;
}
appendToLog("\n(Pass 3/3) Determining optimal n-gram range...\n");
/* DEPRECATED
// Search through min, max for max F-measure ratio
/////////////////////////////////////
int iBestMin = -1;
int iBestMax = -1;
double dBestPerformance = 0.0;
iCurCnt = 0;
iTotal = ((iMaxSymbolSize - iMinSymbolSize + 1) * (iMaxSymbolSize + iMinSymbolSize)) / 2;
/////////////////////////////////////
for (int iMin = iMinSymbolSize; iMin <= iMaxSymbolSize; iMin++) {
for (int iMax = iMin; iMax <= iMaxSymbolSize; iMax++) {
double dSymbolsSum = 0.0;
double dNonSymbolsSum = 0.0;
for (int iCnt=iMin; iCnt<=iMax; iCnt++) {
// Give higher weight to higher order n-gram matches
dSymbolsSum += jinsect.utils.sumFromTo(iMin, iCnt) * dSymbolsPerSize.getValue(iCnt) *
(dSymbolSizes.getValue(iCnt) / dSymbolSizes.sumOfValues()) ;
// Use log for non-symbols
dNonSymbolsSum += dNonSymbolsPerSize.getValue(iCnt) *
(dSymbolSizes.getValue(iCnt) / dSymbolSizes.sumOfValues())
/ sgOverallGraph.getAlphabet().size();
}
double dCurPerformance = (double)dSymbolsSum / (1.0 + dNonSymbolsSum);
if (dCurPerformance > dBestPerformance) {
// New max performance found
iBestMax = iMax;
iBestMin = iMin;
dBestPerformance = dCurPerformance;
}
fStatus.setStatus("(Pass 3/3) Determining optimal n-gram range...",
(double)iCurCnt++ / iTotal);
}
}
*/
NGramSizeEstimator nseEstimator = new NGramSizeEstimator(dSymbolsPerSize, dNonSymbolsPerSize);
IntegerPair p = nseEstimator.getOptimalRange();
// Log proposed value
appendToLog("\nProposed n-gram sizes:" + p.first() + "," +
p.second());
fStatus.setStatus("Determining optimal distance...", 0.0);
// DEPRECATED: double dDev = determineDistanceDeviation();
DistanceEstimator de = new DistanceEstimator(dSymbolsPerSize, dNonSymbolsPerSize, nseEstimator);
// System.err.println("MinNGram MaxNGram BestDist");
// for (int iMinNGram = 1; iMinNGram <= 10; iMinNGram++) {
// for (int iMaxNGram = iMinNGram; iMaxNGram <= 10; iMaxNGram++) {
// int iBestDist = de.getOptimalDistance(1, 20,
// iMinNGram, iMaxNGram);
// // System.err.println(String.format("%d %d %d", iMinNGram, iMaxNGram, iBestDist));
// }
// }
int iBestDist = de.getOptimalDistance(1, nseEstimator.getMaxRank()*2,
p.first(), p.second());
fStatus.setStatus("Determining optimal distance...", 1.0);
appendToLog("\nOptimal distance:" + iBestDist);
appendToLog("======DONE=====\n");
} finally {
sgOverallGraph.removeNotificationListener();
}
}
private void DoCreateNGramGraph() {
final gr.demokritos.iit.jinsect.gui.StatusFrame fStatus = new gr.demokritos.iit.jinsect.gui.StatusFrame();
int Levels = Math.max(NGramSizeSldr.getValue(), 2);
sgOverallGraph = new SymbolicGraph(1, Levels); // Init graph with a min of 2
siIndex = new SemanticIndex(sgOverallGraph);
try {
siIndex.MeaningExtractor = new LocalWordNetMeaningExtractor();
}
catch (IOException ioe) {
siIndex.MeaningExtractor = null; // Use default
}
// sgOverallGraph.setNotificationListener(new NotificationAdapter() {
// public void Notify(Object oSender, Object oParameters) {
// //fStatus.setStatus("Graph size " + ((SymbolicGraph)sgOverallGraph).getEdgesCount(), 0.0);
// System.out.println("Graph size " + ((SymbolicGraph)sgOverallGraph).getEdgesCount() + " Current NGram Size:" + oParameters);
// }
// });
//
try
{
// Get categories and files from disk
DocumentSet dsSet = new DocumentSet(FilePathEdt.getText(), 1.0);
dsSet.createSets(true, (double)CorpusPercentSld.getValue() / 100);
int iCurCnt, iTotal;
String sFile = "";
Iterator iIter = dsSet.getTrainingSet().iterator();
iTotal = dsSet.getTrainingSet().size();
if (iTotal == 0) // No documents to import
{
appendToLog("No input documents.\n");
appendToLog("======DONE=====\n");
return;
}
iCurCnt = 0;
// Create overall document
cdDoc = new DistributionDocument[Levels];
for (int iCnt=0; iCnt < Levels; iCnt++)
cdDoc[iCnt] = new DistributionDocument(1, MinLevel + iCnt); // For all windows
fStatus.setVisible(true);
while (iIter.hasNext())
{
sFile = ((CategorizedFileEntry)iIter.next()).getFileName();
appendToLog("Loading file..." + sFile);
fStatus.setStatus("Loading file..." + sFile, (double)iCurCnt / iTotal);
for (int iCnt=0; iCnt < cdDoc.length; iCnt++) {
cdDoc[iCnt].loadDataStringFromFile(sFile,false);
}
sgOverallGraph.loadFromFile(sFile);
appendToLog("Loaded file..."+ sFile);
fStatus.setStatus("Loaded file..." + sFile, (double)++iCurCnt / iTotal);
Thread.yield();
}
/*
appendToLog("Graphs created");
for (int iCnt=0; iCnt < cdDoc.length; iCnt++) {
appendToLog("***GRAPH LEVEL "+String.valueOf(iCnt) + "***");
//appendToLog("======GRAPH:=====\n" + cdDoc[iCnt].toString());
}
*/
appendToLog("======DONE=====\n");
} finally {
sgOverallGraph.removeNotificationListener();
fStatus.dispose();
}
}
private void CreateNGramGraphBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_CreateNGramGraphBtnActionPerformed
Thread t = new Thread() {
@Override
public void run() {
DoCreateNGramGraph();
unregisterThread(this);
}
};
registerThread(t);
t.setPriority(Math.min(Thread.MAX_PRIORITY, t.getPriority() + 1)); // Increased priority
t.start();
}//GEN-LAST:event_CreateNGramGraphBtnActionPerformed
private void SelectInputFileBtnActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_SelectInputFileBtnActionPerformed
// Select a dir
JFileChooser fc = new JFileChooser();
fc.setCurrentDirectory((FilePathEdt.getText().length() == 0) ? new java.io.File(".") : new java.io.File(FilePathEdt.getText()));
fc.setSelectedFile(fc.getCurrentDirectory());
fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
int iRet = fc.showOpenDialog(this);
if (iRet == JFileChooser.APPROVE_OPTION)
FilePathEdt.setText(fc.getSelectedFile().getAbsolutePath());
}//GEN-LAST:event_SelectInputFileBtnActionPerformed
private static String[] splitStringByDelimiterPoints(String sStr, Integer[] iRes) {
ArrayList alRes = new ArrayList();
// For every split point get substring
for (int iCnt=0; iCnt < iRes.length; iCnt++) {
if (iCnt == 0)
alRes.add(sStr.substring(0, iRes[iCnt]));
else
alRes.add(sStr.substring(iRes[iCnt - 1], iRes[iCnt]));
}
// Add last part
if (iRes.length > 0)
alRes.add(sStr.substring(iRes[iRes.length - 1]));
else
alRes.add(sStr); // No splitting
String[] sRes = new String[alRes.size()]; // n split points => n+1 string parts
alRes.toArray(sRes);
return sRes;
}
private synchronized final void setDelimiters(SortedMap smDelims) {
String sRes = "";
Iterator iIter = smDelims.keySet().iterator();
while (iIter.hasNext()) {
Object oNext = smDelims.get(iIter.next());
sRes = oNext.toString() + sRes; // Reverse
}
try {
DelimitersEdt.setText(sRes);
}
catch (Exception e) {
System.err.println("Cannot update edit (probably due to strange encoding). Continuing...");
}
Delims.clear();
Delims.putAll(smDelims);
}
public synchronized final SortedMap getDelimiters() {
return Delims;
}
/**
* @param args the command line arguments
*/
public static void main(String args[]) {
if (args.length == 0)
java.awt.EventQueue.invokeLater(new Runnable() {
public void run() {
new NGramCorrelationForm().setVisible(true);
}
});
else {
StatusConsole sc = new StatusConsole(80);
Hashtable hSwitches = gr.demokritos.iit.jinsect.utils.parseCommandLineSwitches(args);
int iOutputNGramMinSize = Integer.valueOf(gr.demokritos.iit.jinsect.utils.getSwitch(hSwitches,
"outputSymbolsOfMinSize", "-1")).intValue();
int iOutputNGramMaxSize = Integer.valueOf(gr.demokritos.iit.jinsect.utils.getSwitch(hSwitches,
"outputSymbolsOfMaxSize", String.valueOf(iOutputNGramMinSize + 10))).intValue();
String sSymbolFile = gr.demokritos.iit.jinsect.utils.getSwitch(hSwitches, "symbolFile",
"Symbols.tmp");
boolean bRightToLeftText = Boolean.valueOf(gr.demokritos.iit.jinsect.utils.getSwitch(
hSwitches, "rightToLeft",String.valueOf(false))).booleanValue();
if (iOutputNGramMinSize > 0) {
// Print out symbols
int iCount = printOutSymbolsOfSize(iOutputNGramMinSize, iOutputNGramMaxSize, sSymbolFile, new
StreamOutputConsole(System.out, false));
System.out.println("Total: " + iCount);
}
else {
if (bRightToLeftText) {
System.err.println("Performing right to left analysis...");
}
// else execute extraction
NGramCorrelationForm nfMain = new NGramCorrelationForm();
nfMain.RightToLeftText = bRightToLeftText;
String sInputDir = gr.demokritos.iit.jinsect.utils.getSwitch(hSwitches, "dir", "./DUC/reducedModels2005/");
nfMain.FilePathEdt.setText(sInputDir);
nfMain.analyseCorpus(sc);
nfMain.dispose();
}
}
}
private Set getSymbolsByProbabilities(String sText, IStatusDisplayer fStatus) {
StringBuffer sbSubStr = new StringBuffer();
TreeSet tsRes = new TreeSet();
Date dStartTime = new Date();
// For every character
for (int iCnt = 0; iCnt < sText.length(); iCnt++) {
String sNextChar = sText.substring(iCnt, iCnt+1);
// If the probability of a suffix char given a prefix is higher than
// random, the suffix is considered part of the prefix.
if ((sbSubStr.length() == 0) ||
(getProbabilityOfStringInText(sbSubStr.toString(), sNextChar) >
getProbabilityOfStringInRandomText(sbSubStr.toString(), sNextChar)))
sbSubStr.append(sText.charAt(iCnt));
else
// else end the existing symbol, adding it to the returned set and start a new one
{
tsRes.add(sbSubStr.toString());
// DEBUG LINES
// System.err.println("Found symbol:" + sbSubStr.toString());
// appendToLog("Found symbol:" + sbSubStr.toString());
//////////////
sbSubStr = new StringBuffer(sNextChar);
}
Date dCurTime = new Date();
long lRemaining = (sText.length() - iCnt + 1) *
(long)((double)(dCurTime.getTime() - dStartTime.getTime()) / iCnt);
String sRemaining = String.format(" - Remaining: %40s\r",
gr.demokritos.iit.jinsect.utils.millisToMinSecString(lRemaining));
fStatus.setStatus("Determining corpus symbols..." + sRemaining, (double)iCnt / sText.length());
}
// Add the final symbol, if not empty.
if (sbSubStr.length() > 0)
tsRes.add(sbSubStr.toString());
return tsRes;
}
static public int printOutSymbolsOfSize(int iMinSize, int iMaxSize, String sSymbolsFile,
IStatusDisplayer isStatus) {
// Get symbol set
Set sSymbols = null;
File fPreviousSymbols = new File(sSymbolsFile);
boolean bSymbolsLoadedOK = false;
int iCurCnt = 0;
if (fPreviousSymbols.exists()) {
try {
FileInputStream fis = new FileInputStream(fPreviousSymbols);
ObjectInputStream ois = new ObjectInputStream(fis);
sSymbols = (Set)ois.readObject();
ois.close();
bSymbolsLoadedOK = true;
} catch (FileNotFoundException ex) {
ex.printStackTrace(System.err);
} catch (IOException ex) {
ex.printStackTrace(System.err);
// Reset distros
} catch (ClassNotFoundException ex) {
ex.printStackTrace(System.err);
}
Iterator iSymbolIter = sSymbols.iterator();
int iTotalSize = sSymbols.size();
while (iSymbolIter.hasNext()) {
String sCur = (String)iSymbolIter.next();
int iLen = sCur.length();
if ((iLen >= iMinSize) && (iLen <= iMaxSize)) {
isStatus.setStatus(sCur,(double)iCurCnt / iTotalSize);
iCurCnt++;
}
}
}
return iCurCnt;
}
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton AnalyseCorpusBtn;
private javax.swing.JButton BreakDownBtn;
private javax.swing.JButton BreakFileDownBtn;
private javax.swing.JButton CancelAllBtn;
private javax.swing.JButton ClearGraphsBtn;
private javax.swing.JButton CompareFilesBtn;
private javax.swing.JSlider CorpusPercentSld;
private javax.swing.JButton CreateNGramGraphBtn;
private javax.swing.JTextField DelimitersEdt;
private javax.swing.JTextField FilePathEdt;
private javax.swing.JButton FindPathBtn;
private javax.swing.JTextArea LogMemo;
private javax.swing.JButton LookUpTermBtn;
private javax.swing.JSlider NGramSizeSldr;
private javax.swing.JLabel SecondInputFileLbl;
private javax.swing.JButton SelectInputFileBtn;
private javax.swing.JTextField SelectInputFileEdt;
private javax.swing.JTextField SelectSecondInputFileEdt;
private javax.swing.JButton SelectSecondTestFileBtn;
private javax.swing.JButton SelectTestFileBtn;
private javax.swing.JTextField TermEdt;
private javax.swing.JButton jButton1;
private javax.swing.JLabel jLabel1;
private javax.swing.JLabel jLabel2;
private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4;
private javax.swing.JLabel jLabel5;
private javax.swing.JLabel jLabel6;
private javax.swing.JScrollPane jScrollPane1;
// End of variables declaration//GEN-END:variables
}
