Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[1802] trunk/src/dl-learner/org/dllearner
|
From: <jen...@us...> - 2009-06-16 16:35:15
|
Revision: 1802
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=1802&view=rev
Author: jenslehmann
Date: 2009-06-16 16:35:07 +0000 (Tue, 16 Jun 2009)
Log Message:
-----------
wrote performance evaluation
Modified Paths:
--------------
trunk/src/dl-learner/org/dllearner/core/configurators/ClassLearningProblemConfigurator.java
trunk/src/dl-learner/org/dllearner/core/configurators/ComponentFactory.java
trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java
trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyEngineering.java
trunk/src/dl-learner/org/dllearner/utilities/statistics/Stat.java
Added Paths:
-----------
trunk/src/dl-learner/org/dllearner/core/configurators/PelletReasonerConfigurator.java
Modified: trunk/src/dl-learner/org/dllearner/core/configurators/ClassLearningProblemConfigurator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/core/configurators/ClassLearningProblemConfigurator.java 2009-06-16 09:03:15 UTC (rev 1801)
+++ trunk/src/dl-learner/org/dllearner/core/configurators/ClassLearningProblemConfigurator.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -72,6 +72,15 @@
public String getType() {
return (String) ComponentManager.getInstance().getConfigOptionValue(classLearningProblem, "type") ;
}
+/**
+* useApproximations whether to use stochastic approximations for computing accuracy.
+* mandatory: false| reinit necessary: true
+* default value: true
+* @return boolean
+**/
+public boolean getUseApproximations() {
+return (Boolean) ComponentManager.getInstance().getConfigOptionValue(classLearningProblem, "useApproximations") ;
+}
/**
* @param classToDescribe class of which a description should be learned.
@@ -90,6 +99,15 @@
ComponentManager.getInstance().applyConfigEntry(classLearningProblem, "type", type);
reinitNecessary = true;
}
+/**
+* @param useApproximations whether to use stochastic approximations for computing accuracy.
+* mandatory: false| reinit necessary: true
+* default value: true
+**/
+public void setUseApproximations(boolean useApproximations) {
+ComponentManager.getInstance().applyConfigEntry(classLearningProblem, "useApproximations", useApproximations);
+reinitNecessary = true;
+}
/**
* true, if this component needs reinitializsation.
Modified: trunk/src/dl-learner/org/dllearner/core/configurators/ComponentFactory.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/core/configurators/ComponentFactory.java 2009-06-16 09:03:15 UTC (rev 1801)
+++ trunk/src/dl-learner/org/dllearner/core/configurators/ComponentFactory.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -45,6 +45,7 @@
import org.dllearner.reasoning.FastInstanceChecker;
import org.dllearner.reasoning.FastRetrievalReasoner;
import org.dllearner.reasoning.OWLAPIReasoner;
+import org.dllearner.reasoning.PelletReasoner;
/**
* automatically generated, do not edit manually.
@@ -118,6 +119,14 @@
}
/**
+* @param knowledgeSource see KnowledgeSource
+* @return a component ready for initialization PelletReasoner
+**/
+public static PelletReasoner getPelletReasoner(Set<KnowledgeSource> knowledgeSource) {
+return PelletReasonerConfigurator.getPelletReasoner(knowledgeSource);
+}
+
+/**
* @param classToDescribe class of which a description should be learned
* @param reasoningService see ReasoningService
* @return a component ready for initialization ClassLearningProblem
Added: trunk/src/dl-learner/org/dllearner/core/configurators/PelletReasonerConfigurator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/core/configurators/PelletReasonerConfigurator.java (rev 0)
+++ trunk/src/dl-learner/org/dllearner/core/configurators/PelletReasonerConfigurator.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -0,0 +1,66 @@
+/**
+ * Copyright (C) 2007-2008, Jens Lehmann
+ *
+ * This file is part of DL-Learner.
+ *
+ * DL-Learner is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * DL-Learner is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.dllearner.core.configurators;
+
+import java.util.Set;
+import org.dllearner.core.ComponentManager;
+import org.dllearner.core.KnowledgeSource;
+import org.dllearner.reasoning.PelletReasoner;
+
+/**
+* automatically generated, do not edit manually.
+* run org.dllearner.scripts.ConfigJavaGenerator to update
+**/
+public class PelletReasonerConfigurator implements Configurator {
+
+private boolean reinitNecessary = false;
+@SuppressWarnings("unused")
+
+private PelletReasoner pelletReasoner;
+
+/**
+* @param pelletReasoner see PelletReasoner
+**/
+public PelletReasonerConfigurator(PelletReasoner pelletReasoner){
+this.pelletReasoner = pelletReasoner;
+}
+
+/**
+* @param knowledgeSource see knowledgeSource
+* @return PelletReasoner
+**/
+public static PelletReasoner getPelletReasoner(Set<KnowledgeSource> knowledgeSource) {
+PelletReasoner component = ComponentManager.getInstance().reasoner(PelletReasoner.class, knowledgeSource);
+return component;
+}
+
+
+
+/**
+* true, if this component needs reinitializsation.
+* @return boolean
+**/
+public boolean isReinitNecessary(){
+return reinitNecessary;
+}
+
+
+}
Modified: trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java 2009-06-16 09:03:15 UTC (rev 1801)
+++ trunk/src/dl-learner/org/dllearner/learningproblems/ClassLearningProblem.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -31,6 +31,7 @@
import org.dllearner.core.LearningProblem;
import org.dllearner.core.ReasonerComponent;
import org.dllearner.core.configurators.ClassLearningProblemConfigurator;
+import org.dllearner.core.options.BooleanConfigOption;
import org.dllearner.core.options.ConfigOption;
import org.dllearner.core.options.StringConfigOption;
import org.dllearner.core.options.URLConfigOption;
@@ -57,6 +58,8 @@
// approximation of accuracy +- 0.05 %
private static final double approx = 0.05;
+ private boolean useApproximations;
+
// factor for higher weight on coverage (needed for subclass learning)
private double coverageFactor;
@@ -80,7 +83,9 @@
options.add(classToDescribeOption);
StringConfigOption type = new StringConfigOption("type", "whether to learn an equivalence class or super class axiom","equivalence"); // or domain/range of a property.
type.setAllowedValues(new String[] {"equivalence", "superClass"}); // , "domain", "range"});
- options.add(type);
+ options.add(type);
+ BooleanConfigOption approx = new BooleanConfigOption("useApproximations", "whether to use stochastic approximations for computing accuracy", true);
+ options.add(approx);
return options;
}
@@ -91,6 +96,7 @@
@Override
public void init() throws ComponentInitException {
classToDescribe = new NamedClass(configurator.getClassToDescribe().toString());
+ useApproximations = configurator.getUseApproximations();
if(!reasoner.getNamedClasses().contains(classToDescribe)) {
throw new ComponentInitException("The class \"" + configurator.getClassToDescribe() + "\" does not exist. Make sure you spelled it correctly.");
@@ -201,9 +207,16 @@
@Override
public double getAccuracyOrTooWeak(Description description, double noise) {
- // instead of using the standard operation, we use optimisation
- // and approximation here
-
+ if(useApproximations) {
+ return getAccuracyOrTooWeakApprox(description, noise);
+ } else {
+ return getAccuracyOrTooWeakExact(description, noise);
+ }
+ }
+
+ // instead of using the standard operation, we use optimisation
+ // and approximation here
+ public double getAccuracyOrTooWeakApprox(Description description, double noise) {
// we abort when there are too many uncovered positives
int maxNotCovered = (int) Math.ceil(noise*classInstances.size());
int instancesCovered = 0;
@@ -245,7 +258,7 @@
// if the mean is greater than the required minimum, we can accept;
// we also accept if the interval is small and close to the minimum
// (worst case is to accept a few inaccurate descriptions)
- if(mean > noise || (upperBorderA > mean && size < 0.03)) {
+ if(mean > 1-noise || (upperBorderA > mean && size < 0.03)) {
instancesCovered = (int) (instancesCovered/(double)total * classInstances.size());
upperEstimateA = (int) (upperBorderA * classInstances.size());
lowerEstimateA = (int) (lowerBorderA * classInstances.size());
@@ -255,7 +268,7 @@
// reject only if the upper border is far away (we are very
// certain not to lose a potential solution)
- if(upperBorderA + 0.1 < noise) {
+ if(upperBorderA + 0.1 < 1-noise) {
return -1;
}
}
@@ -328,21 +341,49 @@
// MonitorFactory.add("estimatedB","count", estimatedB ? 1 : 0);
// MonitorFactory.add("bInstances","count", testsPerformed);
- return getAccuracy(coverage, protusion);
- }
+ return getAccuracy(coverage, protusion);
+ }
- @Deprecated
- public double getAccuracyOrTooWeakStandard(Description description, double minAccuracy) {
- // since we have to perform a retrieval operation anyway, we cannot easily
- // get a benefit from the accuracy limit
- double accuracy = getAccuracy(description);
- if(accuracy >= minAccuracy) {
- return accuracy;
- } else {
+ public double getAccuracyOrTooWeakExact(Description description, double noise) {
+ // overhang
+ int additionalInstances = 0;
+ for(Individual ind : superClassInstances) {
+ if(reasoner.hasType(description, ind)) {
+ additionalInstances++;
+ }
+ }
+
+ // coverage
+ int coveredInstances = 0;
+ for(Individual ind : classInstances) {
+ if(reasoner.hasType(description, ind)) {
+ coveredInstances++;
+ }
+ }
+
+ double coverage = coveredInstances/(double)classInstances.size();
+
+ if(coverage < 1 - noise) {
return -1;
}
+
+ double protusion = additionalInstances == 0 ? 0 : coveredInstances/(double)(coveredInstances+additionalInstances);
+
+ return getAccuracy(coverage, protusion);
}
+// @Deprecated
+// public double getAccuracyOrTooWeakStandard(Description description, double minAccuracy) {
+// // since we have to perform a retrieval operation anyway, we cannot easily
+// // get a benefit from the accuracy limit
+// double accuracy = getAccuracy(description);
+// if(accuracy >= minAccuracy) {
+// return accuracy;
+// } else {
+// return -1;
+// }
+// }
+
// computes accuracy from coverage and protusion (changing this function may
// make it necessary to change the appoximation too)
private double getAccuracy(double coverage, double protusion) {
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyEngineering.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyEngineering.java 2009-06-16 09:03:15 UTC (rev 1801)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyEngineering.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -23,6 +23,7 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.net.URL;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Iterator;
@@ -41,6 +42,7 @@
import org.dllearner.core.LearningProblemUnsupportedException;
import org.dllearner.core.ReasonerComponent;
import org.dllearner.core.configurators.CELOEConfigurator;
+import org.dllearner.core.owl.Description;
import org.dllearner.core.owl.Individual;
import org.dllearner.core.owl.NamedClass;
import org.dllearner.core.owl.Thing;
@@ -48,6 +50,7 @@
import org.dllearner.learningproblems.ClassLearningProblem;
import org.dllearner.learningproblems.EvaluatedDescriptionClass;
import org.dllearner.reasoning.FastInstanceChecker;
+import org.dllearner.reasoning.OWLAPIReasoner;
import org.dllearner.utilities.statistics.Stat;
/**
@@ -65,6 +68,8 @@
public class OntologyEngineering {
private static double minAccuracy = 0.85;
+
+ private static double noisePercent = 5.0;
private static int minInstanceCount = 3;
@@ -72,6 +77,12 @@
private static DecimalFormat df = new DecimalFormat();
+ // for performance measurements and development
+ private static boolean autoMode = false;
+ private static boolean useFastInstanceChecker = true;
+ private static boolean useApproximations = true;
+ private static boolean computeApproxDiff = false;
+
@SuppressWarnings("unchecked")
public static void main(String[] args) throws ComponentInitException,
LearningProblemUnsupportedException, IOException {
@@ -83,15 +94,25 @@
System.out.println("You need to give an OWL file as argument.");
System.exit(0);
}
- File owlFile = new File(args[0]);
+
ComponentManager cm = ComponentManager.getInstance();
// load OWL in reasoner
OWLFile ks = cm.knowledgeSource(OWLFile.class);
- ks.getConfigurator().setUrl(owlFile.toURI().toURL());
+ if(args[0].startsWith("http")) {
+ ks.getConfigurator().setUrl(new URL(args[0]));
+ } else {
+ File owlFile = new File(args[0]);
+ ks.getConfigurator().setUrl(owlFile.toURI().toURL());
+ }
ks.init();
- ReasonerComponent reasoner = cm.reasoner(FastInstanceChecker.class, ks);
+ ReasonerComponent reasoner = null;
+ if(useFastInstanceChecker) {
+ reasoner = cm.reasoner(FastInstanceChecker.class, ks);
+ } else {
+ reasoner = cm.reasoner(OWLAPIReasoner.class, ks);
+ }
reasoner.init();
System.out.println("Loaded ontology " + args[0] + ".");
@@ -103,6 +124,7 @@
int classCandidatesCount = 0;
Stat instanceCountStat = new Stat();
Stat classExpressionTestsStat = new Stat();
+ Stat approxDiffStat = new Stat();
// equivalence classes
int candidatesAboveThresholdCount = 0;
@@ -168,6 +190,7 @@
+ algorithmRuntimeInSeconds + " seconds)");
lp.getConfigurator().setType("superClass");
}
+ lp.getConfigurator().setUseApproximations(useApproximations);
lp.init();
CELOE celoe = cm.learningAlgorithm(CELOE.class, lp, reasoner);
@@ -175,7 +198,7 @@
cf.setUseNegation(false);
cf.setValueFrequencyThreshold(3);
cf.setMaxExecutionTimeInSeconds(algorithmRuntimeInSeconds);
- cf.setNoisePercentage(0.05);
+ cf.setNoisePercentage(noisePercent);
cf.setMaxNrOfResults(10);
celoe.init();
@@ -214,6 +237,15 @@
List<EvaluatedDescriptionClass> suggestionsList = new LinkedList<EvaluatedDescriptionClass>(
suggestions.descendingSet());
+ if(computeApproxDiff) {
+ for(EvaluatedDescription ed : suggestionsList) {
+ Description d = ed.getDescription();
+ double approx = lp.getAccuracyOrTooWeakApprox(d, noisePercent/(double)100);
+ double exact = lp.getAccuracyOrTooWeakExact(d, noisePercent/(double)100);
+ approxDiffStat.addNumber(approx-exact);
+ }
+ }
+
int nr = 0;
for (EvaluatedDescription suggestion : suggestionsList) {
System.out.println(nr
@@ -243,11 +275,15 @@
List<String> allowedInputs = Arrays.asList(inputs);
String input;
- do {
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
- input = br.readLine();
- } while (!allowedInputs.contains(input));
-
+ if(autoMode) {
+ input = "n";
+ } else {
+ do {
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+ input = br.readLine();
+ } while (!allowedInputs.contains(input));
+ }
+
userInputProtocol += input;
if (input.equals("m")) {
@@ -340,6 +376,9 @@
System.out.println("classes with at least " + minInstanceCount + " instances: "
+ classCandidatesCount);
System.out.println("class expressions tested: " + classExpressionTestsStat.prettyPrint(""));
+ if(computeApproxDiff) {
+ System.out.println("approximation difference: " + approxDiffStat.prettyPrint());
+ }
System.out.println();
System.out.println("statistics for equivalence axioms:");
Modified: trunk/src/dl-learner/org/dllearner/utilities/statistics/Stat.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/utilities/statistics/Stat.java 2009-06-16 09:03:15 UTC (rev 1801)
+++ trunk/src/dl-learner/org/dllearner/utilities/statistics/Stat.java 2009-06-16 16:35:07 UTC (rev 1802)
@@ -127,6 +127,10 @@
return max;
}
+ public String prettyPrint() {
+ return prettyPrint("");
+ }
+
public String prettyPrint(String unit) {
if(sum > 0) {
DecimalFormat df = new DecimalFormat();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
