Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[2228] trunk
|
From: <lor...@us...> - 2010-08-02 19:45:33
|
Revision: 2228
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=2228&view=rev
Author: lorenz_b
Date: 2010-08-02 19:45:26 +0000 (Mon, 02 Aug 2010)
Log Message:
-----------
Removed more warnings.
Updated latest Protege libs.
Added 2 reasoner test cases.
Added agreement measure.
Modified Paths:
--------------
trunk/doc/configOptions.txt
trunk/lib/protege/org.protege.editor.core.application.jar
trunk/lib/protege/org.protege.editor.owl.jar
trunk/src/dl-learner/org/dllearner/Info.java
trunk/src/dl-learner/org/dllearner/core/configurators/OWLAPIReasonerConfigurator.java
trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyChecker.java
trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java
trunk/src/dl-learner/org/dllearner/test/JenaBug.java
Added Paths:
-----------
trunk/src/dl-learner/org/dllearner/test/HermitTest.java
trunk/src/dl-learner/org/dllearner/test/PelletBug3.java
trunk/src/dl-learner/org/dllearner/utilities/statistics/RawAgreement.java
Removed Paths:
-------------
trunk/src/dl-learner/org/dllearner/test/IncrementalInconsistencyFinder.java
Modified: trunk/doc/configOptions.txt
===================================================================
--- trunk/doc/configOptions.txt 2010-08-02 14:51:20 UTC (rev 2227)
+++ trunk/doc/configOptions.txt 2010-08-02 19:45:26 UTC (rev 2228)
@@ -191,8 +191,8 @@
conf file usage: reasoner = fastInstanceChecker;
option name: reasonerType
-description: FaCT++ or Pellet to dematerialize
-allowed values: String [fact, pellet]
+description: FaCT++, HermiT or Pellet to dematerialize
+allowed values: String [fact, hermit, pellet]
default value: pellet
conf file usage: fastInstanceChecker.reasonerType = pellet;
@@ -215,8 +215,8 @@
conf file usage: reasoner = owlAPIReasoner;
option name: reasonerType
-description: FaCT++ or Pellet, which means "pellet" or "fact"
-allowed values: String [fact, pellet]
+description: FaCT++,, HermiT or Pellet, which means "fact", "hermit" or "pellet"
+allowed values: String [fact, hermit, pellet]
default value: pellet
conf file usage: owlAPIReasoner.reasonerType = pellet;
@@ -290,7 +290,25 @@
default value: true
conf file usage: classLearning.checkConsistency = true;
+option name: maxExecutionTimeInSeconds
+description: algorithm will stop after specified seconds
+allowed values: int
+default value: 10
+conf file usage: classLearning.maxExecutionTimeInSeconds = 10;
+option name: betaSC
+description: Higher values of beta rate recall higher than precision or in other words, covering the instances of the class to describe is more important even at the cost of covering additional instances. The actual implementation depends on the selected heuristic. This values is used only for super class learning.
+allowed values: double
+default value: 3.0
+conf file usage: classLearning.betaSC = 3.0;
+
+option name: betaEq
+description: Higher values of beta rate recall higher than precision or in other words, covering the instances of the class to describe is more important even at the cost of covering additional instances. The actual implementation depends on the selected heuristic. This values is used only for equivalence class learning.
+allowed values: double
+default value: 1.0
+conf file usage: classLearning.betaEq = 1.0;
+
+
component: pos neg learning problem (org.dllearner.learningproblems.PosNegLPStandard)
=====================================================================================
@@ -564,7 +582,25 @@
default value: false
conf file usage: celoe.reuseExistingDescription = false;
+option name: writeSearchTree
+description: specifies whether to write a search tree
+allowed values: boolean
+default value: false
+conf file usage: celoe.writeSearchTree = false;
+option name: searchTreeFile
+description: file to use for the search tree
+allowed values: String []
+default value: log/searchTree.txt
+conf file usage: celoe.searchTreeFile = log/searchTree.txt;
+
+option name: replaceSearchTree
+description: specifies whether to replace the search tree in the log file after each run or append the new search tree
+allowed values: boolean
+default value: false
+conf file usage: celoe.replaceSearchTree = false;
+
+
component: standard EL learning algorithm (org.dllearner.algorithms.el.ELLearningAlgorithm)
===========================================================================================
Modified: trunk/lib/protege/org.protege.editor.core.application.jar
===================================================================
(Binary files differ)
Modified: trunk/lib/protege/org.protege.editor.owl.jar
===================================================================
(Binary files differ)
Modified: trunk/src/dl-learner/org/dllearner/Info.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/Info.java 2010-08-02 14:51:20 UTC (rev 2227)
+++ trunk/src/dl-learner/org/dllearner/Info.java 2010-08-02 19:45:26 UTC (rev 2228)
@@ -3,6 +3,6 @@
package org.dllearner;
public class Info {
- public static final String build = "2010-03-15";
+ public static final String build = "2010-07-21";
}
\ No newline at end of file
Modified: trunk/src/dl-learner/org/dllearner/core/configurators/OWLAPIReasonerConfigurator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/core/configurators/OWLAPIReasonerConfigurator.java 2010-08-02 14:51:20 UTC (rev 2227)
+++ trunk/src/dl-learner/org/dllearner/core/configurators/OWLAPIReasonerConfigurator.java 2010-08-02 19:45:26 UTC (rev 2228)
@@ -53,7 +53,7 @@
}
/**
-* reasonerType FaCT++ or Pellet, which means "pellet" or "fact".
+* reasonerType FaCT++,, HermiT or Pellet, which means "pellet" or "fact".
* mandatory: false| reinit necessary: true
* default value: pellet
* @return String
@@ -63,7 +63,7 @@
}
/**
-* @param reasonerType FaCT++ or Pellet, which means "pellet" or "fact".
+* @param reasonerType FaCT++,, HermiT or Pellet, which means "pellet" or "fact".
* mandatory: false| reinit necessary: true
* default value: pellet
**/
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyChecker.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyChecker.java 2010-08-02 14:51:20 UTC (rev 2227)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/OntologyChecker.java 2010-08-02 19:45:26 UTC (rev 2228)
@@ -77,7 +77,6 @@
private static long reasonerTaskTimeoutInMinutes = 10;
- @SuppressWarnings("unchecked")
public static void main(String[] args) throws ComponentInitException, MalformedURLException {
Map<String, Integer> ontologyRelClassCountMap = new HashMap<String, Integer>();
Set<String> inconsistentOntologies = new HashSet<String>();
@@ -243,17 +242,17 @@
}
- @SuppressWarnings("unchecked")
- static Map sortByValue(Map map) {
- List list = new LinkedList(map.entrySet());
- Collections.sort(list, new Comparator() {
- public int compare(Object o1, Object o2) {
- return ((Comparable) ((Map.Entry) (o1)).getValue()).compareTo(((Map.Entry) (o2)).getValue());
+
+ private static Map<String, Integer> sortByValue(Map<String, Integer> map) {
+ List<Entry<String, Integer>> list = new LinkedList<Entry<String, Integer>>(map.entrySet());
+ Collections.sort(list, new Comparator<Entry<String, Integer>>() {
+ public int compare(Entry<String, Integer> o1, Entry<String, Integer> o2) {
+ return o1.getValue().compareTo(o2.getValue());
}
});
- Map result = new LinkedHashMap();
- for (Iterator it = list.iterator(); it.hasNext();) {
- Map.Entry entry = (Map.Entry) it.next();
+ Map<String, Integer> result = new LinkedHashMap<String, Integer>();
+ for (Iterator<Entry<String, Integer>> it = list.iterator(); it.hasNext();) {
+ Entry<String, Integer> entry = it.next();
result.put(entry.getKey(), entry.getValue());
}
return result;
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java 2010-08-02 14:51:20 UTC (rev 2227)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java 2010-08-02 19:45:26 UTC (rev 2228)
@@ -12,6 +12,7 @@
import java.net.URL;
import java.text.DecimalFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -22,15 +23,20 @@
import java.util.Set;
import java.util.Map.Entry;
+import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.dllearner.core.owl.Individual;
+import org.apache.log4j.SimpleLayout;
import org.dllearner.core.owl.NamedClass;
+import org.dllearner.learningproblems.ClassLearningProblem;
import org.dllearner.learningproblems.EvaluatedDescriptionClass;
-import org.dllearner.utilities.statistics.FleissKappa;
+import org.dllearner.reasoning.FastInstanceChecker;
+import org.dllearner.utilities.owl.ConceptComparator;
+import org.dllearner.utilities.statistics.RawAgreement;
import org.dllearner.utilities.statistics.Stat;
import org.semanticweb.owlapi.apibinding.OWLManager;
import org.semanticweb.owlapi.model.IRI;
+import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.model.OWLOntologyCreationException;
import org.semanticweb.owlapi.model.OWLOntologyManager;
@@ -40,73 +46,196 @@
private Hashtable<NamedClass, Map<EvaluatedDescriptionClass, Integer>> userInputMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> fastEquivalenceStandardMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> fastEquivalenceFMeasureMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> fastEquivalencePredaccMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> fastEquivalenceGenFMeasureMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> fastEquivalenceJaccardMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> fastStandardMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> fastFMeasureMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> fastPredaccMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> fastGenFMeasureMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> fastJaccardMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> owlEquivalenceStandardMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> owlEquivalenceFMeasureMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> owlEquivalencePredaccMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> owlEquivalenceGenFMeasureMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> owlEquivalenceJaccardMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> owlStandardMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> owlFMeasureMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> owlPredaccMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> owlGenFMeasureMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> owlJaccardMap;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> defaultEquivalenceMap;
+ private Map<NamedClass, List<EvaluatedDescriptionClass>> defaultMap;
+
+ private StringBuilder latexStats;
+ private StringBuilder latexMetrics;
+ @SuppressWarnings("unused")
+ private Map<NamedClass, Map<EvaluatedDescriptionClass, Integer>> incMap;
+
+ List<Stat> defaultFractions = new ArrayList<Stat>();
+ List<Stat> owlStandardFractions = new ArrayList<Stat>();
+ List<Stat> owlFMeasureFractions = new ArrayList<Stat>();
+ List<Stat> owlGenFMeasureFractions = new ArrayList<Stat>();
+ List<Stat> owlPredaccFractions = new ArrayList<Stat>();
+ List<Stat> owlJaccardFractions = new ArrayList<Stat>();
+ List<Stat> fastStandardFractions = new ArrayList<Stat>();
+ List<Stat> fastFMeasureFractions = new ArrayList<Stat>();
+ List<Stat> fastGenFMeasureFractions = new ArrayList<Stat>();
+ List<Stat> fastPredaccFractions = new ArrayList<Stat>();
+ List<Stat> fastJaccardFractions = new ArrayList<Stat>();
+ List<Stat> defaultFractionsBest = new ArrayList<Stat>();
+ List<Stat> owlStandardFractionsBest = new ArrayList<Stat>();
+ List<Stat> owlFMeasureFractionsBest = new ArrayList<Stat>();
+ List<Stat> owlGenFMeasureFractionsBest = new ArrayList<Stat>();
+ List<Stat> owlPredaccFractionsBest = new ArrayList<Stat>();
+ List<Stat> owlJaccardFractionsBest = new ArrayList<Stat>();
+ List<Stat> fastStandardFractionsBest = new ArrayList<Stat>();
+ List<Stat> fastFMeasureFractionsBest = new ArrayList<Stat>();
+ List<Stat> fastGenFMeasureFractionsBest = new ArrayList<Stat>();
+ List<Stat> fastPredaccFractionsBest = new ArrayList<Stat>();
+ List<Stat> fastJaccardFractionsBest = new ArrayList<Stat>();
+ int[] count = new int[6];
+ int[] bestCount = new int[6];
+ int best = 6;
-
-
- private Map<NamedClass, List<EvaluatedDescriptionClass>> equivalentSuggestions;
- private Map<NamedClass, List<EvaluatedDescriptionClass>> superSuggestions;
-
- private Map<NamedClass, String> equivalentInput;
- private Map<NamedClass, String> superInput;
+ int classesCount = 0;
- private Map<NamedClass, List<Integer>> equivalentRating;
- private Map<NamedClass, List<Integer>> superRating;
+ private static final int RATER = 4;
+ private static final double MIN_ACCURACY = 0.9;
+
+ int defaultMissedImprovementsCount = 0;
+ int owlStandardMissedImprovementsCount = 0;
+ int owlFMeasureMissedImprovementsCount = 0;
+ int owlGenFMeasureMissedImprovementsCount = 0;
+ int owlJaccardMissedImprovementsCount = 0;
+ int owlPredaccMissedImprovementsCount = 0;
+ int fastStandardMissedImprovementsCount = 0;
+ int fastFMeasureMissedImprovementsCount = 0;
+ int fastGenFMeasureMissedImprovementsCount = 0;
+ int fastJaccardMissedImprovementsCount = 0;
+ int fastPredaccMissedImprovementsCount = 0;
+
+ double defaultSuggestions = 0;
+ double owlStandardSuggestions = 0;
+ double owlFMeasureSuggestions = 0;
+ double owlGenFMeasureSuggestions = 0;
+ double owlJaccardSuggestions = 0;
+ double owlPredaccSuggestions = 0;
+ double fastStandardSuggestions = 0;
+ double fastFMeasureSuggestions = 0;
+ double fastGenFMeasureSuggestions = 0;
+ double fastJaccardSuggestions = 0;
+ double fastPredaccSuggestions = 0;
+
+ private Set<Stat> defaultPositionStats = new HashSet<Stat>();
+ private Set<Stat> defaultAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> defaultAccStats = new HashSet<Stat>();
+ private Set<Stat> defaultHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> owlStandardPositionStats = new HashSet<Stat>();
+ private Set<Stat> owlStandardAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> owlStandardAccStats = new HashSet<Stat>();
+ private Set<Stat> owlStandardHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> owlFMeasurePositionStats = new HashSet<Stat>();
+ private Set<Stat> owlFMeasureAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> owlFMeasureAccStats = new HashSet<Stat>();
+ private Set<Stat> owlFMeasureHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> owlGenFMeasurePositionStats = new HashSet<Stat>();
+ private Set<Stat> owlGenFMeasureAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> owlGenFMeasureAccStats = new HashSet<Stat>();
+ private Set<Stat> owlGenFMeasureHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> owlJaccardPositionStats = new HashSet<Stat>();
+ private Set<Stat> owlJaccardAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> owlJaccardAccStats = new HashSet<Stat>();
+ private Set<Stat> owlJaccardHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> owlPredaccPositionStats = new HashSet<Stat>();
+ private Set<Stat> owlPredaccAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> owlPredaccAccStats = new HashSet<Stat>();
+ private Set<Stat> owlPredaccHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> fastStandardPositionStats = new HashSet<Stat>();
+ private Set<Stat> fastStandardAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> fastStandardAccStats = new HashSet<Stat>();
+ private Set<Stat> fastStandardHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> fastFMeasurePositionStats = new HashSet<Stat>();
+ private Set<Stat> fastFMeasureAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> fastFMeasureAccStats = new HashSet<Stat>();
+ private Set<Stat> fastFMeasureHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> fastGenFMeasurePositionStats = new HashSet<Stat>();
+ private Set<Stat> fastGenFMeasureAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> fastGenFMeasureAccStats = new HashSet<Stat>();
+ private Set<Stat> fastGenFMeasureHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> fastJaccardPositionStats = new HashSet<Stat>();
+ private Set<Stat> fastJaccardAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> fastJaccardAccStats = new HashSet<Stat>();
+ private Set<Stat> fastJaccardHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ private Set<Stat> fastPredaccPositionStats = new HashSet<Stat>();
+ private Set<Stat> fastPredaccAddInstancesCountStats = new HashSet<Stat>();
+ private Set<Stat> fastPredaccAccStats = new HashSet<Stat>();
+ private Set<Stat> fastPredaccHiddenInconsistenciesCountStats = new HashSet<Stat>();
+
+ @SuppressWarnings("unused")
+ private Stat defaultSuggestionsCountStat = new Stat();
+ private Stat owlStandardSuggestionsCountStat = new Stat();
+ private Stat owlFMeasureSuggestionsCountStat = new Stat();
+ private Stat owlGenFMeasureSuggestionsCountStat = new Stat();
+ private Stat owlPredaccSuggestionsCountStat = new Stat();
+ private Stat owlJaccardSuggestionsCountStat = new Stat();
+ private Stat fastStandardSuggestionsCountStat = new Stat();
+ private Stat fastFMeasureSuggestionsCountStat = new Stat();
+ private Stat fastGenFMeasureSuggestionsCountStat = new Stat();
+ private Stat fastPredaccSuggestionsCountStat = new Stat();
+ private Stat fastJaccardSuggestionsCountStat = new Stat();
- private StringBuilder latexStats;
- private StringBuilder latexMetrics;
-
- // general stats
- private Stat acceptedGlobalStat = new Stat();
- private Stat rejectedGlobalStat = new Stat();
- private Stat failedGlobalStat = new Stat();
-
- // equivalent class stats
- private Stat acceptedStat = new Stat();
- private Stat rejectedStat = new Stat();
- private Stat failedStat = new Stat();
- private Set<Stat> moreInstancesCountStats = new HashSet<Stat>();
- private Set<Stat> accStats = new HashSet<Stat>();
- private Set<Stat> accSelectedStats = new HashSet<Stat>();
- private Set<Stat> accAboveThresholdStats = new HashSet<Stat>();
- private Set<Stat> positionStats = new HashSet<Stat>();
-
- // super class stats
- private Stat acceptedStatSC = new Stat();
- private Stat rejectedStatSC = new Stat();
- private Stat failedStatSC = new Stat();
- private Set<Stat> moreInstancesCountStatsSC = new HashSet<Stat>();
- private Set<Stat> accStatsSC = new HashSet<Stat>();
- private Set<Stat> accSelectedStatsSC = new HashSet<Stat>();
- private Set<Stat> accAboveThresholdStatsSC = new HashSet<Stat>();
- private Set<Stat> positionStatsSC = new HashSet<Stat>();
-
- private int suggestionListsCount;
- private int logicalAxiomCount;
private OWLOntology ont;
//matrix for Fleiss Kappa input
private short[][] mat;
- private List<NamedClass> equivalentLists;
- private List<NamedClass> superLists;
-
+
+ private Set<NamedClass> classesToEvaluate = new HashSet<NamedClass>();
+
+ private ConceptComparator c = new ConceptComparator();
+
+ private File directory;
+
+ OWLDataFactory factory ;
+ ClassLearningProblem lp;
+ FastInstanceChecker rc;
+
public StatsGenerator(File directory) {
+ this.directory = directory;
+
+ for(int i = 0; i < 6; i++){
+ defaultFractions.add(new Stat());
+ owlStandardFractions.add(new Stat());
+ owlFMeasureFractions.add(new Stat());
+ owlGenFMeasureFractions.add(new Stat());
+ owlPredaccFractions.add(new Stat());
+ owlJaccardFractions.add(new Stat());
+ fastStandardFractions.add(new Stat());
+ fastFMeasureFractions.add(new Stat());
+ fastGenFMeasureFractions.add(new Stat());
+ fastPredaccFractions.add(new Stat());
+ fastJaccardFractions.add(new Stat());
+
+ defaultFractionsBest.add(new Stat());
+ owlStandardFractionsBest.add(new Stat());
+ owlFMeasureFractionsBest.add(new Stat());
+ owlGenFMeasureFractionsBest.add(new Stat());
+ owlPredaccFractionsBest.add(new Stat());
+ owlJaccardFractionsBest.add(new Stat());
+ fastStandardFractionsBest.add(new Stat());
+ fastFMeasureFractionsBest.add(new Stat());
+ fastGenFMeasureFractionsBest.add(new Stat());
+ fastPredaccFractionsBest.add(new Stat());
+ fastJaccardFractionsBest.add(new Stat());
+ }
+ count();
// begin latex table with headers
beginOntologyMetricsTable();
beginStatsTable();
@@ -114,251 +243,1301 @@
for (File suggestionFile : directory.listFiles(new ResultFileFilter())) {
loadLearnResults(suggestionFile);
loadOntology(suggestionFile);
-// resetStats();
+ computeClassesToEvaluate();
// for each user evaluation input file
for (File inputFile : directory.listFiles(new NameFilter(suggestionFile))) {
loadUserInput(inputFile);
- evaluateOWLStandard();
-// makeSingleStat();
+ evaluateUserInput();
}
// add row to the metrics latex table for current ontology
addOntologyMetricsTableRow();
- // add row to the stats latex table for current ontology
-// addStatsTableRow();
- break;
+// break;
}
+// showMeasureValues();
+ computeAverageSuggestionsPerClass();
+ showMeasureFractions();
+ generateBestMeasureFractionsTable();
+ generateMetricsTable();
+ computeFleissKappa(directory);
// end latex tables
endTables();
printLatexCode();
}
- private void evaluateOWLStandard(){
- Stat value = new Stat();
+ private void evaluateUserInput(){
+ evaluateDefault();
+ evaluateFastFMeasure();
+ evaluateFastGenFMeasure();
+ evaluateFastJaccard();
+ evaluateFastPredacc();
+ evaluateFastStandard();
+ evaluateOWLFMeasure();
+ evaluateOWLGenFMeasure();
+ evaluateOWLJaccard();
+ evaluateOWLPredacc();
+ evaluateOWLStandard();
+ }
+
+ private void computeAverageSuggestionsPerClass(){
+ for (File suggestionFile : directory.listFiles(new ResultFileFilter())) {
+ loadLearnResults(suggestionFile);
+ computeClassesToEvaluate();
+ classesCount += classesToEvaluate.size();
+ defaultSuggestions += getSuggestionsCount(defaultMap);
+ owlStandardSuggestions += getSuggestionsCount(owlStandardMap);
+ owlFMeasureSuggestions += getSuggestionsCount(owlFMeasureMap);
+ owlGenFMeasureSuggestions += getSuggestionsCount(owlGenFMeasureMap);
+ owlPredaccSuggestions += getSuggestionsCount(owlPredaccMap);
+ owlJaccardSuggestions += getSuggestionsCount(owlJaccardMap);
+ fastStandardSuggestions += getSuggestionsCount(fastStandardMap);
+ fastFMeasureSuggestions += getSuggestionsCount(fastFMeasureMap);
+ fastGenFMeasureSuggestions += getSuggestionsCount(fastGenFMeasureMap);
+ fastPredaccSuggestions += getSuggestionsCount(fastPredaccMap);
+ fastJaccardSuggestions += getSuggestionsCount(fastJaccardMap);
+ }
+ System.out.println(classesCount);
+ }
+
+ private void count(){
+
+ for (File suggestionFile : directory.listFiles(new ResultFileFilter())) {
+ loadLearnResults(suggestionFile);
+ computeClassesToEvaluate();
+ for(NamedClass nc : classesToEvaluate){
+ System.out.println(nc + " : " + fastStandardMap.get(nc).size());
+ owlStandardSuggestionsCountStat.addNumber(owlStandardMap.get(nc).size());
+ owlFMeasureSuggestionsCountStat.addNumber(owlFMeasureMap.get(nc).size());
+ owlGenFMeasureSuggestionsCountStat.addNumber(owlGenFMeasureMap.get(nc).size());
+ owlPredaccSuggestionsCountStat.addNumber(owlPredaccMap.get(nc).size());
+ owlJaccardSuggestionsCountStat.addNumber(owlJaccardMap.get(nc).size());
+ fastStandardSuggestionsCountStat.addNumber(fastStandardMap.get(nc).size());
+ fastFMeasureSuggestionsCountStat.addNumber(fastFMeasureMap.get(nc).size());
+ fastGenFMeasureSuggestionsCountStat.addNumber(fastGenFMeasureMap.get(nc).size());
+ fastPredaccSuggestionsCountStat.addNumber(fastPredaccMap.get(nc).size());
+ fastJaccardSuggestionsCountStat.addNumber(fastJaccardMap.get(nc).size());
+ }
+ }
+ }
+
+ private int getSuggestionsCount(Map<NamedClass, List<EvaluatedDescriptionClass>> map){
+ int suggestionsCount = 0;
+ for(Entry<NamedClass, List<EvaluatedDescriptionClass>> entry : map.entrySet()){
+ if(!classesToEvaluate.contains(entry.getKey())){
+ continue;
+ }
+ suggestionsCount += entry.getValue().size();
+ }
+
+ return suggestionsCount;
+ }
+
+ private void showMeasureFractions(){
+ DecimalFormat df = new DecimalFormat( "0.00" );
+ StringBuilder sb = new StringBuilder();
+ sb.append("\\begin{tabular}{| l | c | c | c | c | c | c |} \n");
+ sb.append(" & ");
+ sb.append("\\rotatebox{90}{Improvement} & ");
+ sb.append("\\rotatebox{90}{Equal quality (+)} & ");
+ sb.append("\\rotatebox{90}{Equal quality (-)} & ");
+ sb.append("\\rotatebox{90}{Inferior} & ");
+ sb.append("\\rotatebox{90}{Not acceptable} & ");
+ sb.append("\\rotatebox{90}{Error}");
+ sb.append(" \\\\\n");
+ sb.append("\\hline\n");
+
+
+ sb.append("Default & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(defaultFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("OWLAPI Reasoner Standard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlStandardFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("OWLAPI Reasoner FMeasure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlFMeasureFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("OWLAPI Reasoner GenFMeasure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlGenFMeasureFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("OWLAPI Reasoner Predacc & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlPredaccFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("OWLAPI Reasoner Jaccard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlJaccardFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("FastInstanceChecker Standard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastStandardFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("FastInstanceChecker FMeasure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastFMeasureFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("FastInstanceChecker GenFMeasure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastGenFMeasureFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("FastInstanceChecker Predacc & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastPredaccFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+ sb.append(" \\\\\n");
+ sb.append("FastInstanceChecker Jaccard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastJaccardFractions.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+
+ sb.append(" \\\\\n");
+ sb.append("\\hline\n");
+ sb.append("\\end{tabular}");
+ System.out.println(sb.toString());
+
+ }
+
+ private void generateMetricsTable(){
+
+ DecimalFormat df = new DecimalFormat( "0.00" );
+ DecimalFormat df1 = new DecimalFormat( "0" );
+ StringBuilder sb = new StringBuilder();
+ sb.append("\\begin{tabular}{ l | c | c | c | c | c | c } \n");
+ sb.append("reasoner / heuristic & \n");
+ sb.append("\\rotatebox{90}{missed improvements in \\%} & \n");
+ sb.append("\\rotatebox{90}{avg. suggestions per class} & \n");
+ sb.append("\\begin{sideways}\\makecell[l]{selected position\\\\on suggestion list\\\\(incl.~std.~deviation)}\\end{sideways} & \n");
+ sb.append("\\begin{sideways}\\makecell[l]{avg. accuracy of\\\\selected suggestion in \\%}\\end{sideways} & \n");
+ sb.append("\\begin{sideways}\\makecell[l]{\\#hidden inconsistencies}\\end{sideways} & \n");
+ sb.append("\\rotatebox{90}{\\#add.~instances total}");
+ sb.append(" \\\\\n");
+ sb.append("\\hline\n");
+
+ sb.append("Pellet / R-Measure & ");
+ sb.append(df.format((double)owlStandardMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(owlStandardSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlStandardPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(owlStandardPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlStandardAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlStandardHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(owlStandardAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / F-Measure & ");
+ sb.append(df.format((double)owlFMeasureMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(owlFMeasureSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlFMeasurePositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(owlFMeasurePositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlFMeasureAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlFMeasureHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(owlFMeasureAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / Gen. F-Measure & ");
+ sb.append(df.format((double)owlGenFMeasureMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(owlGenFMeasureSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlGenFMeasurePositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(owlGenFMeasurePositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlGenFMeasureAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlGenFMeasureHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(owlGenFMeasureAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / pred. acc. & ");
+ sb.append(df.format((double)owlPredaccMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(owlPredaccSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlPredaccPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(owlPredaccPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlPredaccAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlPredaccHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(owlPredaccAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / Jaccard & ");
+ sb.append(df.format((double)owlJaccardMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(owlJaccardSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlJaccardPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(owlJaccardPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlJaccardAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(owlJaccardHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(owlJaccardAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / R-Measure & ");
+ sb.append(df.format((double)fastStandardMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(fastStandardSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastStandardPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(fastStandardPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastStandardAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastStandardHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(fastStandardAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / F-Measure & ");
+ sb.append(df.format((double)fastFMeasureMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(fastFMeasureSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastFMeasurePositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(fastFMeasurePositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastFMeasureAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastFMeasureHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(fastFMeasureAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / Gen. F-Measure & ");
+ sb.append(df.format((double)fastGenFMeasureMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(fastGenFMeasureSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastGenFMeasurePositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(fastGenFMeasurePositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastGenFMeasureAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastGenFMeasureHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(fastGenFMeasureAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / pred. acc. & ");
+ sb.append(df.format((double)fastPredaccMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(fastPredaccSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastPredaccPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(fastPredaccPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastPredaccAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastPredaccHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(fastPredaccAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / Jaccard & ");
+ sb.append(df.format((double)fastJaccardMissedImprovementsCount / classesCount * 100));
+ sb.append(" & ");
+ sb.append(df.format(fastJaccardSuggestionsCountStat.getMean()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastJaccardPositionStats).getMean()));
+ sb.append(" $\\pm$ ");
+ sb.append(df.format(new Stat(fastJaccardPositionStats).getStandardDeviation()));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastJaccardAccStats).getMean() * 100));
+ sb.append(" & ");
+ sb.append(df.format(new Stat(fastJaccardHiddenInconsistenciesCountStats).getSum() / 5));
+ sb.append(" & ");
+ sb.append(df1.format(new Stat(fastJaccardAddInstancesCountStats).getSum() / RATER));
+ sb.append(" \\\\\n");
+
+ sb.append("\\hline\n");
+ sb.append("\\end{tabular}");
+
+ System.out.println(sb.toString());
+ }
+
+ private void generateBestMeasureFractionsTable(){
+ DecimalFormat df = new DecimalFormat( "0.00" );
+ StringBuilder sb = new StringBuilder();
+ sb.append("\\begin{tabular}{| l | c | c | c | c | c | c |} \n");
+ sb.append(" & ");
+ sb.append("\\rotatebox{90}{improvement} & ");
+ sb.append("\\rotatebox{90}{equal quality (+)} & ");
+ sb.append("\\rotatebox{90}{equal quality (-)} & ");
+ sb.append("\\rotatebox{90}{inferior} & ");
+ sb.append("\\rotatebox{90}{not acceptable} & ");
+ sb.append("\\rotatebox{90}{error} ");
+ sb.append(" \\\\\n");
+ sb.append("\\hline\n");
+
+ classesCount *= RATER;
+
+// sb.append("Default & ");
+// for(int i = 0; i < 6; i++){
+// sb.append(df.format(defaultFractionsBest.get(i).getMean() * 100));
+// sb.append(" & ");
+// }
+// sb.append(df.format((double)defaultMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(defaultSuggestions/(double)classesCount * 4));
+// sb.append(" \\\\\n");
+
+ sb.append("Pellet / R-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlStandardFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)owlStandardMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(owlStandardSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / F-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlFMeasureFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)owlFMeasureMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(owlFMeasureSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / Gen. F-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlGenFMeasureFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)owlGenFMeasureMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(owlGenFMeasureSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / pred. acc. & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlPredaccFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)owlPredaccMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(owlPredaccSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet / Jaccard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(owlJaccardFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)owlJaccardMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(owlJaccardSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / R-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastStandardFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)fastStandardMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(fastStandardSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / F-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastFMeasureFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)fastFMeasureMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(fastFMeasureSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / Gen. F-Measure & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastGenFMeasureFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)fastGenFMeasureMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(fastGenFMeasureSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / pred. acc. & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastPredaccFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)fastPredaccMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(fastPredaccSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("Pellet FIC / Jaccard & ");
+ for(int i = 0; i < 6; i++){
+ sb.append(df.format(fastJaccardFractionsBest.get(i).getMean() * 100));
+ if(i < 5){
+ sb.append(" & ");
+ }
+ }
+// sb.append(df.format((double)fastJaccardMissedImprovementsCount / classesCount * 100));
+// sb.append(" & ");
+// sb.append(df.format(fastJaccardSuggestions/(double)classesCount * 4));
+ sb.append(" \\\\\n");
+
+ sb.append("\\hline\n");
+ sb.append("\\end{tabular}");
+ System.out.println(sb.toString());
+ }
+
+
+
+ private boolean isImprovementSelected(NamedClass nc){
+ Map<EvaluatedDescriptionClass, Integer> input = userInputMap.get(nc);
+ boolean improvementSelected = false;
+ for(EvaluatedDescriptionClass ec : input.keySet()){
+ for(EvaluatedDescriptionClass ec2 : defaultMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : fastStandardMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : fastFMeasureMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : fastGenFMeasureMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : fastJaccardMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : fastPredaccMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : owlStandardMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : owlFMeasureMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : owlGenFMeasureMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : owlJaccardMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+ for(EvaluatedDescriptionClass ec2 : owlPredaccMap.get(nc)){
+ if(ec2.getAccuracy() < MIN_ACCURACY){
+ continue;
+ }
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ if(input.get(ec) == 1){
+ return true;
+ }
+ }
+ }
+
+ }
+
+ return improvementSelected;
+ }
+
+ private void computeClassesToEvaluate(){
+ classesToEvaluate.clear();
+ for(Entry<NamedClass, List<EvaluatedDescriptionClass>> entry : defaultMap.entrySet()){
+ for(EvaluatedDescriptionClass ec : entry.getValue()){
+ if(ec.getAccuracy() >= MIN_ACCURACY){
+ classesToEvaluate.add(entry.getKey());
+ break;
+ }
+ }
+ }
+ }
+
+
+ private void evaluateDefault(){
Map<EvaluatedDescriptionClass, Integer> input;
- for(Entry<NamedClass, List<EvaluatedDescriptionClass>> entry : owlEquivalenceStandardMap.entrySet()){
- input = userInputMap.get(entry.getKey());
- for(EvaluatedDescriptionClass ec : entry.getValue()){
+ resetBestCount();
+ Stat positionStat = new Stat();
+ Stat moreInstancesCountStat = new Stat();
+ Stat accStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : defaultMap.get(nc)){
for(EvaluatedDescriptionClass ec2 : input.keySet()){
- if(ec.getDescription().equals(ec2.getDescription())){
- value.addNumber(input.get(ec2));
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+ }
}
}
-
}
+ for(int i = 0; i < 6; i++){
+ defaultFractions.get(i).addNumber((double)count[i]/defaultMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ defaultMissedImprovementsCount++;
+ }
+ }
}
- System.out.println(value.getMean());
+ for(int i = 0; i < 6; i++){
+ defaultFractionsBest.get(i).addNumber((double)bestCount[i]/classesToEvaluate.size());
+ }
+ defaultPositionStats.add(positionStat);
+ defaultAccStats.add(accStat);
+ defaultAddInstancesCountStats.add(moreInstancesCountStat);
+ defaultHiddenInconsistenciesCountStats.add(hiddenIncCountStat);
}
-
- private void loadOntology(File file) {
- String ontologyPath = file.toURI().toString().substring(0, file.toURI().toString().lastIndexOf('.')) + ".owl";
- OWLOntologyManager man = OWLManager.createOWLOntologyManager();
- try {
- ont = man.loadOntologyFromOntologyDocument(IRI.create(ontologyPath));
- logicalAxiomCount = ont.getLogicalAxiomCount();
- } catch (OWLOntologyCreationException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+
+ private void evaluateOWLStandard(){
+ Map<EvaluatedDescriptionClass, Integer> input;
+ resetBestCount();
+ Stat positionStat = new Stat();
+ Stat moreInstancesCountStat = new Stat();
+ Stat accStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : owlStandardMap.get(nc)){
+ for(EvaluatedDescriptionClass ec2 : input.keySet()){
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+ }
+ }
+ }
+ }
+ for(int i = 0; i < 6; i++){
+ owlStandardFractions.get(i).addNumber((double)count[i]/owlStandardMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ owlStandardMissedImprovementsCount++;
+ }
+ }
+
}
+ for(int i = 0; i < 6; i++){
+ owlStandardFractionsBest.get(i).addNumber((double)bestCount[i]/classesToEvaluate.size());
+ }
+ owlStandardPositionStats.add(positionStat);
+ owlStandardAccStats.add(accStat);
+ owlStandardAddInstancesCountStats.add(moreInstancesCountStat);
+ owlStandardHiddenInconsistenciesCountStats.add(hiddenIncCountStat);
}
-
- private void resetStats() {
- acceptedGlobalStat = new Stat();
- rejectedGlobalStat = new Stat();
- failedGlobalStat = new Stat();
-
- moreInstancesCountStats.clear();
- accStats.clear();
- accSelectedStats.clear();
- accAboveThresholdStats.clear();
- positionStats.clear();
-
- moreInstancesCountStatsSC.clear();
- accStatsSC.clear();
- accSelectedStatsSC.clear();
- accAboveThresholdStatsSC.clear();
- positionStatsSC.clear();
-
- equivalentLists = new ArrayList<NamedClass>(equivalentSuggestions.keySet());
- superLists = new ArrayList<NamedClass>(superSuggestions.keySet());
- mat = new short[suggestionListsCount][3];
+
+ private void evaluateOWLFMeasure(){
+ Map<EvaluatedDescriptionClass, Integer> input;
+ resetBestCount();
+ Stat positionStat = new Stat();
+ Stat moreInstancesCountStat = new Stat();
+ Stat accStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : owlFMeasureMap.get(nc)){
+ for(EvaluatedDescriptionClass ec2 : input.keySet()){
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+
+ }
+ }
+ }
+ }
+ for(int i = 0; i < 6; i++){
+ owlFMeasureFractions.get(i).addNumber((double)count[i]/owlFMeasureMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ owlFMeasureMissedImprovementsCount++;
+ }
+ }
+
+ }
+ for(int i = 0; i < 6; i++){
+ owlFMeasureFractionsBest.get(i).addNumber((double)bestCount[i]/classesToEvaluate.size());
+ }
+ owlFMeasurePositionStats.add(positionStat);
+ owlFMeasureAccStats.add(accStat);
+ owlFMeasureAddInstancesCountStats.add(moreInstancesCountStat);
+ owlFMeasureHiddenInconsistenciesCountStats.add(hiddenIncCountStat);
}
-
- private void makeSingleStat() {
- // equivalence classes
- int candidatesAboveThresholdCount = 0;
- int missesCount = 0;
- int foundDescriptionCount = 0;
- int noSensibleDescriptionCount = 0;
- int inconsistencyDetected = 0;
- int moreInstancesCount = 0;
- int nonPerfectCount = 0;
+
+ private void evaluateOWLGenFMeasure(){
+ Map<EvaluatedDescriptionClass, Integer> input;
+ resetBestCount();
+ Stat positionStat = new Stat();
Stat moreInstancesCountStat = new Stat();
Stat accStat = new Stat();
- Stat accSelectedStat = new Stat();
- Stat accAboveThresholdStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : owlGenFMeasureMap.get(nc)){
+ for(EvaluatedDescriptionClass ec2 : input.keySet()){
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+ }
+ }
+ }
+ }
+ for(int i = 0; i < 6; i++){
+ owlGenFMeasureFractions.get(i).addNumber((double)count[i]/owlGenFMeasureMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ owlGenFMeasureMissedImprovementsCount++;
+ }
+ }
+
+ }
+ for(int i = 0; i < 6; i++){
+ owlGenFMeasureFractionsBest.get(i).addNumber((double)bestCount[i]/classesToEvaluate.size());
+ }
+ owlGenFMeasurePositionStats.add(positionStat);
+ owlGenFMeasureAccStats.add(accStat);
+ owlGenFMeasureAddInstancesCountStats.add(moreInstancesCountStat);
+ owlGenFMeasureHiddenInconsistenciesCountStats.add(hiddenIncCountStat);
+ }
+
+ private void evaluateOWLJaccard(){
+ Map<EvaluatedDescriptionClass, Integer> input;
+ resetBestCount();
Stat positionStat = new Stat();
-
- // super classes
- int candidatesAboveThresholdCountSC = 0;
- int missesCountSC = 0;
- int foundDescriptionCountSC = 0;
- int noSensibleDescriptionCountSC = 0;
- int inconsistencyDetectedSC = 0;
- int moreInstancesCountSC = 0;
- int nonPerfectCountSC = 0;
- Stat moreInstancesCountStatSC = new Stat();
- Stat accStatSC = new Stat();
- Stat accSelectedStatSC = new Stat();
- Stat accAboveThresholdStatSC = new Stat();
- Stat positionStatSC = new Stat();
- // analysing input for equivalent class expressions
- for (Entry<NamedClass, String> e : equivalentInput.entrySet()) {
- NamedClass currentClass = e.getKey();
- String input = e.getValue();
- if(input.equals("-1")){
- input = "n";
+ Stat moreInstancesCountStat = new Stat();
+ Stat accStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : owlJaccardMap.get(nc)){
+ for(EvaluatedDescriptionClass ec2 : input.keySet()){
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+ }
+ }
+ }
}
- if (input.equals("m")) {
- missesCount++;
- mat[equivalentLists.indexOf(currentClass)][2]++;
- } else if (input.equals("n")) {
- noSensibleDescriptionCount++;
- mat[equivalentLists.indexOf(currentClass)][1]++;
- } else {
- mat[equivalentLists.indexOf(currentClass)][0]++;
- int selectedIndex = Integer.parseInt(input);
- EvaluatedDescriptionClass selectedExpression = equivalentSuggestions.get(currentClass).get(
- selectedIndex);
- double bestAcc = equivalentSuggestions.get(currentClass).get(0).getAccuracy();
- int selectedNr = selectedIndex + 1;
- boolean isConsistent = selectedExpression.isConsistent();
- Set<Individual> addInst = selectedExpression.getAdditionalInstances();
- int additionalInstances = addInst.size();
-
- accSelectedStat.addNumber(selectedExpression.getAccuracy());
- positionStat.addNumber(selectedNr);
- foundDescriptionCount++;
- if (!isConsistent) {
- inconsistencyDetected++;
+ for(int i = 0; i < 6; i++){
+ owlJaccardFractions.get(i).addNumber((double)count[i]/owlJaccardMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ owlJaccardMissedImprovementsCount++;
}
- if (additionalInstances > 0) {
- moreInstancesCount++;
- moreInstancesCountStat.addNumber(additionalInstances);
+ }
+
+ }
+ for(int i = 0; i < 6; i++){
+ owlJaccardFractionsBest.get(i).addNumber((double)bestCount[i]/classesToEvaluate.size());
+ }
+ owlJaccardPositionStats.add(positionStat);
+ owlJaccardAccStats.add(accStat);
+ owlJaccardAddInstancesCountStats.add(moreInstancesCountStat);
+ owlJaccardHiddenInconsistenciesCountStats.add(hiddenIncCountStat);
+ }
+
+ private void evaluateOWLPredacc(){
+ Map<EvaluatedDescriptionClass, Integer> input;
+ resetBestCount();
+ Stat positionStat = new Stat();
+ Stat moreInstancesCountStat = new Stat();
+ Stat accStat = new Stat();
+ Stat hiddenIncCountStat = new Stat();
+ boolean first = true;
+ for(NamedClass nc : classesToEvaluate){
+ first = true;
+ resetCount();
+ input = userInputMap.get(nc);
+ for(EvaluatedDescriptionClass ec : owlPredaccMap.get(nc)){
+ for(EvaluatedDescriptionClass ec2 : input.keySet()){
+ if(c.compare(ec.getDescription(), ec2.getDescription()) == 0){
+ count[input.get(ec2) - 1]++;
+ if(best > input.get(ec2)){
+ best = input.get(ec2);
+ }
+ if(best == 1 && first){
+ first = false;
+ positionStat.addNumber(defaultMap.get(nc).indexOf(ec));
+ moreInstancesCountStat.addNumber(ec.getAdditionalInstances().size());
+ if(!ec.isConsistent()){
+ hiddenIncCountStat.addNumber(1);
+ }
+ }
+ if(input.get(ec2) == 1){
+ accStat.addNumber(ec.getAccuracy());
+ }
+ }
}
- if (bestAcc < 0.9999) {
- nonPerfectCount++;
+ }
+ for(int i = 0; i < 6; i++){
+ owlPredaccFractions.get(i).addNumber((double)count[i]/owlPredaccMap.get(nc).size());
+ }
+ bestCount[best - 1]++;
+ if(best != 1){
+ if(isImprovementSelected(nc)){
+ owlPredaccMissedImprovementsCount++;
}
}
+
}
- acceptedStat.addNumber(foundDescriptionCount);
- rejectedStat.addNumber(noSensibleDescriptionCount);
- failedStat.addNumber(missesCount);
- moreInstancesCountStats.add(moreInstancesCountStat);
- accStats.add(accStat);
- accSelectedStats.add(accSelectedStat);
- accAboveThresholdStats.add(accSelectedStat);
- positionStats.add(positionStat);
-
- // analysing input for super class expressions
- for (Entry<NamedClass, String> e : superInput.entrySet()) {
- NamedClass currentClass = e.getKey();
- if (e.getValue().equals("m")) {
- missesCountSC++;
- mat[superLists.indexOf(currentClass) + equivalentLists.size()][2]++;
- } else if (e.getValue().equals("n")) {
- noSensibleDescriptionCountSC++;
- mat[superLists.indexOf(currentClass) + equivalentLists.size()][1]++;
- } else {
- mat[superLists.indexOf(currentClass) + equivalentLists.size()][0]++;
- int selectedIndex = Integer.pa...
[truncated message content] |
