weka.classifiers.evaluation.ThresholdCurve Java Examples

/**
 * Creates a panel displaying the data.
 *
 * @param data          the plot data
 * @param title	  the title
 * @return              the panel
 * @throws Exception    if plot generation fails
 */
protected static ThresholdVisualizePanel createPanel(Instances data, String title) throws Exception {
  ThresholdVisualizePanel result = new ThresholdVisualizePanel();
  PlotData2D plot = new PlotData2D(data);
  plot.setPlotName(title);
  plot.m_displayAllPoints = true;
  boolean[] connectPoints = new boolean [data.numInstances()];
  for (int cp = 1; cp < connectPoints.length; cp++)
    connectPoints[cp] = true;
  plot.setConnectPoints(connectPoints);
  result.addPlot(plot);
  result.setROCString("AUC: " + Utils.doubleToString(ThresholdCurve.getROCArea(data), 3));
  result.setUpComboBoxes(result.getInstances());
  if (data.attribute(ThresholdCurve.FP_RATE_NAME) != null)
    result.setXIndex(data.attribute(ThresholdCurve.FP_RATE_NAME).index());
  if (data.attribute(ThresholdCurve.TP_RATE_NAME) != null)
    result.setYIndex(data.attribute(ThresholdCurve.TP_RATE_NAME).index());
  return result;
}
 

/**
 * Creates a panel displaying the data.
 *
 * @param data          the plot data
 * @param title	  the title
 * @return              the panel
 * @throws Exception    if plot generation fails
 */
protected static ThresholdVisualizePanel createPanel(Instances data, String title) throws Exception {
  ThresholdVisualizePanel result = new ThresholdVisualizePanel();
  PlotData2D plot = new PlotData2D(data);
  plot.setPlotName(title);
  plot.m_displayAllPoints = true;
  boolean[] connectPoints = new boolean [data.numInstances()];
  for (int cp = 1; cp < connectPoints.length; cp++)
    connectPoints[cp] = true;
  plot.setConnectPoints(connectPoints);
  result.addPlot(plot);
  result.setROCString("PRC area: " + Utils.doubleToString(ThresholdCurve.getPRCArea(data), 3));
  result.setUpComboBoxes(result.getInstances());
  if (data.attribute(ThresholdCurve.RECALL_NAME) != null)
    result.setXIndex(data.attribute(ThresholdCurve.RECALL_NAME).index());
  if (data.attribute(ThresholdCurve.PRECISION_NAME) != null)
    result.setYIndex(data.attribute(ThresholdCurve.PRECISION_NAME).index());
  return result;
}
 

/** Get Data for Plotting PR and ROC curves. */
   public static Instances curveDataMicroAveraged(int Y[][], double P[][]) {
//works with missing

int y[] = MatrixUtils.flatten(Y);
double p[] = MatrixUtils.flatten(P);

double[][] aligned = align(y, p);

y = toIntArray(aligned[0]);
p = aligned[1];

ThresholdCurve curve = new ThresholdCurve();
return curve.getCurve(MLUtils.toWekaPredictions(y,p));
   }
 

public void testp_microa(){
    for(TestMetricObject tmo : tmos){
        
        Instances curvedata = Metrics.curveDataMicroAveraged(tmo.real,
                                                             tmo.pred);

        if (curvedata != null) {

            

        
            assertEquals(tmo.p_microauroc,
                         ThresholdCurve.getROCArea(curvedata), 0.00000001);
            
            assertEquals(tmo.p_microauprc,
                         ThresholdCurve.getPRCArea(curvedata), 0.00000001);
        }
        // TODO What if it is null? For example when all is missing?

    }
}
 

/**
 * Finds the best threshold, this implementation searches for the
 * highest FMeasure. If no FMeasure higher than MIN_VALUE is found,
 * the default threshold of 0.5 is used.
 *
 * @param predictions a <code>FastVector</code> containing the predictions.
 */
protected void findThreshold(FastVector predictions) {

  Instances curve = (new ThresholdCurve()).getCurve(predictions, m_DesignatedClass);

  double low = 1.0;
  double high = 0.0;

  //System.err.println(curve);
  if (curve.numInstances() > 0) {
    Instance maxInst = curve.instance(0);
    double maxValue = 0; 
    int index1 = 0;
    int index2 = 0;
    switch (m_nMeasure) {
      case FMEASURE:
        index1 = curve.attribute(ThresholdCurve.FMEASURE_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case TRUE_POS:
        index1 = curve.attribute(ThresholdCurve.TRUE_POS_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case TRUE_NEG:
        index1 = curve.attribute(ThresholdCurve.TRUE_NEG_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case TP_RATE:
        index1 = curve.attribute(ThresholdCurve.TP_RATE_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case PRECISION:
        index1 = curve.attribute(ThresholdCurve.PRECISION_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case RECALL:
        index1 = curve.attribute(ThresholdCurve.RECALL_NAME).index();
        maxValue = maxInst.value(index1);
        break;
      case ACCURACY:
        index1 = curve.attribute(ThresholdCurve.TRUE_POS_NAME).index();
        index2 = curve.attribute(ThresholdCurve.TRUE_NEG_NAME).index();
        maxValue = maxInst.value(index1) + maxInst.value(index2);
        break;
    }
    int indexThreshold = curve.attribute(ThresholdCurve.THRESHOLD_NAME).index();
    for (int i = 1; i < curve.numInstances(); i++) {
      Instance current = curve.instance(i);
      double currentValue = 0;
      if (m_nMeasure ==  ACCURACY) {
        currentValue= current.value(index1) + current.value(index2);
 } else {
     currentValue= current.value(index1);
 }

 if (currentValue> maxValue) {
     maxInst = current;
     maxValue = currentValue;
 }
 if (m_RangeMode == RANGE_BOUNDS) {
     double thresh = current.value(indexThreshold);
     if (thresh < low) {
  low = thresh;
     }
     if (thresh > high) {
  high = thresh;
     }
 }
    }
    if (maxValue > MIN_VALUE) {
      m_BestThreshold = maxInst.value(indexThreshold);
      m_BestValue = maxValue;
      //System.err.println("maxFM: " + maxFM);
    }
    if (m_RangeMode == RANGE_BOUNDS) {
 m_LowThreshold = low;
 m_HighThreshold = high;
      //System.err.println("Threshold range: " + low + " - " + high);
    }
  }

}
 

/**
     * Helper function, returns macro AUROC (roc = true) or macro RPC (roc = false)
     */
    private static double getMacro(int Y[][], double P[][], boolean roc){

        // works with missing
	int L = Y[0].length;
	double AUC[] = new double[L];

        int missing = 0;
        
	for(int j = 0; j < L; j++) {
	    if(allMissing(MatrixUtils.getCol(Y, j))){
                missing ++;
		continue;
	    }
	    ThresholdCurve curve = new ThresholdCurve();

            double[][] aligned = align(MatrixUtils.getCol(Y, j),
                                       MatrixUtils.getCol(P, j));
            
	    Instances result =
                curve.getCurve(MLUtils.toWekaPredictions(toIntArray(aligned[0]),
                                                         aligned[1]));
            if (roc) {
                AUC[j] = ThresholdCurve.getROCArea(result);
            } else {
                AUC[j] = ThresholdCurve.getPRCArea(result);
            }
//            System.out.println(Arrays.toString(MatrixUtils.getCol(Y, j)));
//            System.out.println(Arrays.toString(MatrixUtils.getCol(P, j)));
//
//            System.out.println(AUC[j]);
        }

        L -= missing;
        
        if (L == 0) {
            return Double.NaN;
        }
        
	return Utils.mean(AUC);

        
    }
 

/**
 * Creates a panel displaying the ROC data.
 *
 * @param data          the threshold curve data
 * @param title         the title of the plot
 * @return              the panel
 * @throws Exception    if plot generation fails
 */
protected ThresholdVisualizePanel createPanel(Instances data, String title) throws Exception {
	ThresholdVisualizePanel result = super.createPanel(data, title);
	result.setROCString("AUC: " + Utils.doubleToString(ThresholdCurve.getROCArea(data), 3));
	result.setUpComboBoxes(result.getInstances());
	return result;
}
 

/**
 * Creates a panel displaying the ROC data.
 *
 * @param data          the threshold curve data
 * @param title         the title of the plot
 * @return              the panel
 * @throws Exception    if plot generation fails
 */
protected ThresholdVisualizePanel createPanel(Instances data, String title) throws Exception {
	ThresholdVisualizePanel result = super.createPanel(data, title);
	result.setROCString("PRC area: " + Utils.doubleToString(ThresholdCurve.getPRCArea(data), 3));
	result.setUpComboBoxes(result.getInstances());
	setComboBoxIndices(data, result);
	return result;
}
 

/** Get Data for Plotting PR and ROC curves. */
   public static Instances curveData(int y[], double p[]) {
// works with missing
double[][] aligned = align(y, p);

y = toIntArray(aligned[0]);
p = aligned[1];



ThresholdCurve curve = new ThresholdCurve();
return curve.getCurve(MLUtils.toWekaPredictions(y,p));
   }
 

/**
 * Returns the name of the default Y column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultYColumn() {
	return ThresholdCurve.PRECISION_NAME;
}
 

/**
 * Returns the name of the default X column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultXColumn() {
	return ThresholdCurve.FP_RATE_NAME;
}
 

/**
 * Returns the name of the default Y column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultYColumn() {
	return ThresholdCurve.TP_RATE_NAME;
}
 

/**
 * Returns the name of the default X column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultXColumn() {
	return ThresholdCurve.RECALL_NAME;
}
 

/**
 * Returns the name of the default Y column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultYColumn() {
	return ThresholdCurve.PRECISION_NAME;
}
 

/**
 * Returns the name of the default X column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultXColumn() {
	return ThresholdCurve.RECALL_NAME;
}
 

/**
 * Returns the name of the default Y column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultYColumn() {
	return ThresholdCurve.PRECISION_NAME;
}
 

/**
 * Returns the name of the default X column to display.
 *
 * @return              the name of the column
 */
protected String getDefaultXColumn() {
	return ThresholdCurve.RECALL_NAME;
}