Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[1965] trunk/src/dl-learner/org/dllearner
|
From: <lor...@us...> - 2009-12-29 20:43:56
|
Revision: 1965
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=1965&view=rev
Author: lorenz_b
Date: 2009-12-29 20:43:47 +0000 (Tue, 29 Dec 2009)
Log Message:
-----------
Added input tracing for rating tables.
Added Fleiss' Kappa as a statistical measure for assessing the reliability of agreement between a fixed number of raters when assigning categorical ratings to a number of items or classifying items.
Modified Paths:
--------------
trunk/src/dl-learner/org/dllearner/scripts/evaluation/EvaluationGUI.java
trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java
Added Paths:
-----------
trunk/src/dl-learner/org/dllearner/utilities/statistics/FleissKappa.java
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/EvaluationGUI.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/EvaluationGUI.java 2009-12-29 20:03:55 UTC (rev 1964)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/EvaluationGUI.java 2009-12-29 20:43:47 UTC (rev 1965)
@@ -24,6 +24,7 @@
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -157,6 +158,9 @@
private Map<NamedClass, String> selectedEquivalenceMap = new HashMap<NamedClass, String>();
private Map<NamedClass, String> selectedSuperMap = new HashMap<NamedClass, String>();
+
+ private Map<NamedClass, List<Integer>> equivalentClassListRating = new HashMap<NamedClass, List<Integer>>();
+ private Map<NamedClass, List<Integer>> superClassListRating = new HashMap<NamedClass, List<Integer>>();
public EvaluationGUI(File input) throws ComponentInitException, MalformedURLException,
LearningProblemUnsupportedException {
@@ -515,10 +519,12 @@
@Override
public void actionPerformed(ActionEvent e) {
+ traceInput();
if (e.getActionCommand().equals("next")) {
+
NamedClass nc = classesTable.getSelectedClass(currentClassIndex);
if(!showingMultiTables){
- traceInput();
+
}
if (showingMultiTables && showingEquivalentSuggestions) {
if (defaultSuperMap.get(nc) != null) {
@@ -584,9 +590,6 @@
resetSingleTablePanel();
} else if (e.getActionCommand().equals("finish")) {
- if(!showingMultiTables){
- traceInput();
- }
closeDialog();
saveInput();
}
@@ -601,24 +604,43 @@
private void traceInput(){
NamedClass currentClass = classesTable.getSelectedClass(currentClassIndex);
- if(alternateSuggestionCheckBox.isSelected()){
- if(showingEquivalentSuggestions){
- selectedEquivalenceMap.put(currentClass, "m");
+ if (!showingMultiTables) {
+ if (alternateSuggestionCheckBox.isSelected()) {
+ if (showingEquivalentSuggestions) {
+ selectedEquivalenceMap.put(currentClass, "m");
+ } else {
+ selectedSuperMap.put(currentClass, "m");
+ }
+ } else if (noSuggestionCheckBox.isSelected()) {
+ if (showingEquivalentSuggestions) {
+ selectedEquivalenceMap.put(currentClass, "n");
+ } else {
+ selectedSuperMap.put(currentClass, "n");
+ }
} else {
- selectedSuperMap.put(currentClass, "m");
+ int position = defaultTab.getSelectedPosition() - 1;
+ if (showingEquivalentSuggestions) {
+ selectedEquivalenceMap.put(currentClass, String.valueOf(position));
+ } else {
+ selectedSuperMap.put(currentClass, String.valueOf(position));
+ }
}
- } else if(noSuggestionCheckBox.isSelected()){
- if(showingEquivalentSuggestions){
- selectedEquivalenceMap.put(currentClass, "n");
- } else {
- selectedSuperMap.put(currentClass, "n");
- }
} else {
- int position = defaultTab.getSelectedPosition() - 1;
+ List<Integer> ratingList = new ArrayList<Integer>();
+ ratingList.add(tab1.getRatingValue());
+ ratingList.add(tab2.getRatingValue());
+ ratingList.add(tab3.getRatingValue());
+ ratingList.add(tab4.getRatingValue());
+ ratingList.add(tab5.getRatingValue());
+ ratingList.add(tab6.getRatingValue());
+ ratingList.add(tab7.getRatingValue());
+ ratingList.add(tab8.getRatingValue());
+ ratingList.add(tab9.getRatingValue());
+ ratingList.add(tab10.getRatingValue());
if(showingEquivalentSuggestions){
- selectedEquivalenceMap.put(currentClass, String.valueOf(position));
+ equivalentClassListRating.put(currentClass, ratingList);
} else {
- selectedSuperMap.put(currentClass, String.valueOf(position));
+ superClassListRating.put(currentClass, ratingList);
}
}
}
@@ -639,6 +661,9 @@
o.writeObject(selectedEquivalenceMap);
o.writeObject(selectedSuperMap);
+ o.writeObject(equivalentClassListRating);
+ o.writeObject(superClassListRating);
+
o.flush();
o.close();
} catch (IOException e) {
Modified: trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java 2009-12-29 20:03:55 UTC (rev 1964)
+++ trunk/src/dl-learner/org/dllearner/scripts/evaluation/StatsGenerator.java 2009-12-29 20:43:47 UTC (rev 1965)
@@ -381,6 +381,7 @@
equivalentInput = (Map<NamedClass, String>) o.readObject();
superInput = (Map<NamedClass, String>) o.readObject();
+
} catch (IOException e) {
System.err.println(e);
} catch (ClassNotFoundException e) {
Added: trunk/src/dl-learner/org/dllearner/utilities/statistics/FleissKappa.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/utilities/statistics/FleissKappa.java (rev 0)
+++ trunk/src/dl-learner/org/dllearner/utilities/statistics/FleissKappa.java 2009-12-29 20:43:47 UTC (rev 1965)
@@ -0,0 +1,152 @@
+package org.dllearner.utilities.statistics;
+
+import java.util.Arrays;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * Computes the Fleiss' Kappa value as described in (Fleiss, 1971).
+ * Fleiss' Kappa is a statistical measure for assessing the reliability of agreement between
+ * a fixed number of raters when assigning categorical ratings to a number of items or classifying items.
+ */
+public class FleissKappa
+{
+ private static Logger logger = Logger.getLogger(FleissKappa.class);
+
+ /**
+ * Example from Wikipedia article
+ */
+ public static void main(String[] args)
+ {
+ Logger.getRootLogger().setLevel(Level.DEBUG);
+ short[][] mat = new short[][]
+ {
+ {0,0,0,0,14},
+ {0,2,6,4,2},
+ {0,0,3,5,6},
+ {0,3,9,2,0},
+ {2,2,8,1,1},
+ {7,7,0,0,0},
+ {3,2,6,3,0},
+ {2,5,3,2,2},
+ {6,5,2,1,0},
+ {0,2,2,3,7}
+ } ;
+
+ float kappa = computeKappa(mat) ;
+ System.out.println(getInterpretation(kappa));
+ }
+
+ /**
+ * Computes the Kappa value
+ * @param n Number of rating per subjects (number of human raters)
+ * @param mat Matrix[subjects][categories]
+ * @return The Kappa value
+ */
+ public static float computeKappa(short[][] mat) {
+ final int n = checkEachLineCount(mat);
+ final int N = mat.length;
+ final int k = mat[0].length;
+
+ logger.debug(n + " raters.");
+ logger.debug(N + " subjects.");
+ logger.debug(k + " categories.");
+
+ // Computing columns p[]
+ float[] p = new float[k];
+ for (int j = 0; j < k; j++) {
+ p[j] = 0;
+ for (int i = 0; i < N; i++) {
+ p[j] += mat[i][j];
+ }
+ p[j] /= N * n;
+ }
+ logger.debug("p = " + Arrays.toString(p));
+
+ // Computing rows P[]
+ float[] P = new float[N];
+ for (int i = 0; i < N; i++) {
+ P[i] = 0;
+ for (int j = 0; j < k; j++) {
+ P[i] += mat[i][j] * mat[i][j];
+ }
+ P[i] = (P[i] - n) / (n * (n - 1));
+ }
+ logger.debug("P = " + Arrays.toString(P));
+
+ // Computing Pbar
+ float Pbar = 0;
+ for (float Pi : P) {
+ Pbar += Pi;
+ }
+ Pbar /= N;
+ logger.debug("Pbar = " + Pbar);
+
+ // Computing PbarE
+ float PbarE = 0;
+ for (float pj : p) {
+ PbarE += pj * pj;
+ }
+ logger.debug("PbarE = " + PbarE);
+
+ final float kappa = (Pbar - PbarE) / (1 - PbarE);
+ logger.debug("kappa = " + kappa);
+
+ return kappa;
+ }
+
+ /**
+ * Assert that each line has a constant number of ratings
+ * @param mat The matrix checked
+ * @return The number of ratings
+ * @throws IllegalArgumentException If lines contain different number of ratings
+ */
+ private static int checkEachLineCount(short[][] mat) {
+ int n = 0;
+ boolean firstLine = true;
+
+ for (short[] line : mat) {
+ int count = 0;
+ for (short cell : line) {
+ count += cell;
+ }
+ if (firstLine) {
+ n = count;
+ firstLine = false;
+ }
+ if (n != count) {
+ throw new IllegalArgumentException("Line count != " + n + " (n value).");
+ }
+ }
+ return n;
+ }
+
+ /**
+ * Landis and Koch (1977) gave the following table for interpreting κ values. This table is
+ * however by no means universally accepted; They supplied no evidence to support it, basing it
+ * instead on personal opinion. It has been noted that these guidelines may be more harmful than
+ * helpful, as the number of categories and subjects will affect the magnitude of the value.
+ * The kappa will be higher when there are fewer categories. (Wikipedia)
+ * @param kappa The Kappa value
+ */
+ public static String getInterpretation(float kappa){
+ String interpretation = "";
+ if(kappa < 0){
+ interpretation = "Poor agreement";
+ } else if(0 <= kappa && kappa <= 0.2 ){
+ interpretation = "Slight agreement";
+ } else if(0.2 < kappa && kappa <= 0.4 ){
+ interpretation = "Fair agreement";
+ } else if(0.4 <= kappa && kappa <= 0.6 ){
+ interpretation = "Moderate agreement";
+ } else if(0.6 <= kappa && kappa <= 0.8 ){
+ interpretation = "Substantial agreement";
+ } else if(0.8 <= kappa && kappa <= 1 ){
+ interpretation = "Almost perfect agreement";
+ }
+
+ return interpretation;
+ }
+}
+
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
