Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[2201] trunk/src/dl-learner/org/dllearner
|
From: <jen...@us...> - 2010-07-15 14:15:27
|
Revision: 2201
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=2201&view=rev
Author: jenslehmann
Date: 2010-07-15 14:15:20 +0000 (Thu, 15 Jul 2010)
Log Message:
-----------
unit tests for heuristics in super class learning
Modified Paths:
--------------
trunk/src/dl-learner/org/dllearner/algorithms/celoe/CELOE.java
trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java
trunk/src/dl-learner/org/dllearner/scripts/evaluation/HeuristicOverviewTableGenerator.java
trunk/src/dl-learner/org/dllearner/test/junit/HeuristicTests.java
Modified: trunk/src/dl-learner/org/dllearner/algorithms/celoe/CELOE.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/algorithms/celoe/CELOE.java 2010-07-15 13:18:16 UTC (rev 2200)
+++ trunk/src/dl-learner/org/dllearner/algorithms/celoe/CELOE.java 2010-07-15 14:15:20 UTC (rev 2201)
@@ -31,7 +31,6 @@
import java.util.TreeSet;
import org.apache.log4j.Logger;
-import org.dllearner.algorithms.refinement2.ExampleBasedNode;
import org.dllearner.core.ComponentInitException;
import org.dllearner.core.EvaluatedDescription;
import org.dllearner.core.LearningAlgorithm;
Modified: trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java 2010-07-15 13:18:16 UTC (rev 2200)
+++ trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java 2010-07-15 14:15:20 UTC (rev 2201)
@@ -231,7 +231,7 @@
if(heuristic.equals(HeuristicType.FMEASURE)) {
acc = getFMeasure(recall, precision);
} else if(heuristic.equals(HeuristicType.AMEASURE)) {
- acc = getAccuracy(recall, precision);
+ acc = Heuristics.getAScore(recall, precision, coverageFactor);
} else {
// TODO: some superfluous instance checks are required to compute accuracy =>
// move accuracy computation here if possible
@@ -431,10 +431,10 @@
double size;
if(estimatedA) {
// size = 1/(coverageFactor+1) * (coverageFactor * (upperBorderA-lowerBorderA) + Math.sqrt(upperEstimateA/(upperEstimateA+lowerEstimate)) + Math.sqrt(lowerEstimateA/(lowerEstimateA+upperEstimate)));
- size = heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(upperBorderA, upperEstimateA/(double)(upperEstimateA+lowerEstimate)) - getFMeasure(lowerBorderA, lowerEstimateA/(double)(lowerEstimateA+upperEstimate)) : getAccuracy(upperBorderA, upperEstimateA/(double)(upperEstimateA+lowerEstimate)) - getAccuracy(lowerBorderA, lowerEstimateA/(double)(lowerEstimateA+upperEstimate));
+ size = heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(upperBorderA, upperEstimateA/(double)(upperEstimateA+lowerEstimate)) - getFMeasure(lowerBorderA, lowerEstimateA/(double)(lowerEstimateA+upperEstimate)) : Heuristics.getAScore(upperBorderA, upperEstimateA/(double)(upperEstimateA+lowerEstimate), coverageFactor) - Heuristics.getAScore(lowerBorderA, lowerEstimateA/(double)(lowerEstimateA+upperEstimate),coverageFactor);
} else {
// size = 1/(coverageFactor+1) * (coverageFactor * coverage + Math.sqrt(instancesCovered/(instancesCovered+lowerEstimate)) + Math.sqrt(instancesCovered/(instancesCovered+upperEstimate)));
- size = heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(recall, instancesCovered/(double)(instancesCovered+lowerEstimate)) - getFMeasure(recall, instancesCovered/(double)(instancesCovered+upperEstimate)) : getAccuracy(recall, instancesCovered/(double)(instancesCovered+lowerEstimate)) - getAccuracy(recall, instancesCovered/(double)(instancesCovered+upperEstimate));
+ size = heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(recall, instancesCovered/(double)(instancesCovered+lowerEstimate)) - getFMeasure(recall, instancesCovered/(double)(instancesCovered+upperEstimate)) : Heuristics.getAScore(recall, instancesCovered/(double)(instancesCovered+lowerEstimate),coverageFactor) - Heuristics.getAScore(recall, instancesCovered/(double)(instancesCovered+upperEstimate),coverageFactor);
}
if(size < 0.1) {
@@ -457,7 +457,7 @@
precision = 0;
}
- return heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(recall, precision) : getAccuracy(recall, precision);
+ return heuristic.equals(HeuristicType.FMEASURE) ? getFMeasure(recall, precision) : Heuristics.getAScore(recall, precision, coverageFactor);
} else {
throw new Error("Approximation for " + heuristic + " not implemented.");
@@ -538,14 +538,13 @@
double precision = (additionalInstances + coveredInstances == 0) ? 0 : coveredInstances / (double) (coveredInstances + additionalInstances);
-
if(heuristic.equals(HeuristicType.AMEASURE)) {
// best reachable concept has same recall and precision 1:
// 1/t+1 * (t*r + 1)
if((coverageFactor*recall+1)/(double)(coverageFactor+1) <(1-noise)) {
return -1;
} else {
- return getAccuracy(recall, precision);
+ return Heuristics.getAScore(recall, precision, coverageFactor);
}
} else if(heuristic.equals(HeuristicType.FMEASURE)) {
// best reachable concept has same recall and precision 1:
@@ -718,11 +717,11 @@
// computes accuracy from coverage and protusion (changing this function may
// make it necessary to change the appoximation too) => not the case anymore
- private double getAccuracy(double recall, double precision) {
+// private double getAccuracy(double recall, double precision) {
// return (coverageFactor * coverage + Math.sqrt(protusion)) / (coverageFactor + 1);
// log: changed from precision^^0.5 (root) to precision^^0.8 as the root is too optimistic in some cases
- return (coverageFactor * recall + Math.pow(precision, 0.8)) / (coverageFactor + 1);
- }
+// return (coverageFactor * recall + Math.pow(precision, 0.8)) / (coverageFactor + 1);
+// }
private double getFMeasure(double recall, double precision) {
// balanced F measure
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/HeuristicOverviewTableGenerator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/HeuristicOverviewTableGenerator.java 2010-07-15 13:18:16 UTC (rev 2200)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/HeuristicOverviewTableGenerator.java 2010-07-15 14:15:20 UTC (rev 2201)
@@ -73,7 +73,7 @@
}
private List<Input> inputs;
- private DecimalFormat df = new DecimalFormat();
+ private DecimalFormat df = new DecimalFormat( "0.000000000000" );
public HeuristicOverviewTableGenerator(List<Input> inputs) {
this.inputs = inputs;
@@ -144,7 +144,9 @@
inputs.add(new Input(100, 400, 1000, 100));
inputs.add(new Input(100, 100, 1000, 90));
inputs.add(new Input(100, 50, 1000, 50));
-
+ inputs.add(new Input(20, 20, 100, 10));
+ inputs.add(new Input(20, 40, 100, 10));
+
HeuristicOverviewTableGenerator gen = new HeuristicOverviewTableGenerator(inputs);
gen.printFullTable();
}
Modified: trunk/src/dl-learner/org/dllearner/test/junit/HeuristicTests.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/test/junit/HeuristicTests.java 2010-07-15 13:18:16 UTC (rev 2200)
+++ trunk/src/dl-learner/org/dllearner/test/junit/HeuristicTests.java 2010-07-15 14:15:20 UTC (rev 2201)
@@ -92,6 +92,8 @@
ks.init();
reasoner.init();
+ //// equivalent classes, no noise, no approximations ////
+
// evaluate A2 wrt. A1 using Jaccard
HeuristicTests.configureClassLP(problem, nc[0], "jaccard");
// the value should be 10 (i10-i19) divided by 30 (i0-i29)
@@ -103,12 +105,38 @@
assertEqualsClassLP(problem, nc[1], (10+70)/(double)100);
assertEqualsClassLP(problem, nc[2], (10+50)/(double)100);
+ HeuristicTests.configureClassLP(problem, nc[0], "standard");
+ assertEqualsClassLP(problem, nc[1], 0.5);
+ assertEqualsClassLP(problem, nc[2], 0.375);
+
HeuristicTests.configureClassLP(problem, nc[0], "fmeasure");
// recall = precision = F1-score = 0.5
assertEqualsClassLP(problem, nc[1], 0.5);
// recall = 0.5, precision = 0.25, F1-score = 0.33...
assertEqualsClassLP(problem, nc[2], 1/(double)3);
+ //// super class learning ////
+
+ // Jaccard
+ HeuristicTests.configureClassLP(problem, nc[0], "jaccard", false, false, 0.05);
+ // the value should be 10 (i10-i19) divided by 30 (i0-i29)
+ assertEqualsClassLP(problem, nc[1], 1/(double)3);
+ assertEqualsClassLP(problem, nc[2], 1/(double)5);
+
+ HeuristicTests.configureClassLP(problem, nc[0], "pred_acc", false, false, 0.05);
+ assertEqualsClassLP(problem, nc[1], 5/(double)7);
+ assertEqualsClassLP(problem, nc[2], 4/(double)7);
+
+ HeuristicTests.configureClassLP(problem, nc[0], "standard");
+ assertEqualsClassLP(problem, nc[1], 0.5);
+ assertEqualsClassLP(problem, nc[2], 0.4375);
+
+ HeuristicTests.configureClassLP(problem, nc[0], "fmeasure", false, false, 0.05);
+ // recall = precision = F1-score = 0.5
+ assertEqualsClassLP(problem, nc[1], 0.5);
+ // recall = 0.5, precision = 0.25, F1-score = 0.33...
+ assertEqualsClassLP(problem, nc[2], 0.366025403784);
+
// TODO: generalised F-Measure
// TODO: test approximations
@@ -140,12 +168,13 @@
}
// convencience method to set the learning problem to a desired configuration
- private static void configureClassLP(ClassLearningProblem problem, NamedClass classToDescribe, String accuracyMethod, boolean useApproximations, double approxAccuracy) throws ComponentInitException {
+ private static void configureClassLP(ClassLearningProblem problem, NamedClass classToDescribe, String accuracyMethod, boolean equivalenceLearning, boolean useApproximations, double approxAccuracy) throws ComponentInitException {
try {
problem.getConfigurator().setClassToDescribe(new URL(classToDescribe.getName()));
} catch (MalformedURLException e) {
e.printStackTrace();
}
+ problem.getConfigurator().setType("superClass");
problem.getConfigurator().setAccuracyMethod(accuracyMethod);
problem.getConfigurator().setUseApproximations(useApproximations);
problem.getConfigurator().setApproxAccuracy(approxAccuracy);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
