Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[3868] branches/hmm/components-ext/src
|
From: <ki...@us...> - 2012-11-07 18:13:53
|
Revision: 3868
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=3868&view=rev
Author: kirdie
Date: 2012-11-07 18:13:41 +0000 (Wed, 07 Nov 2012)
Log Message:
-----------
fixed a bug with a comparator used for a treeset with slots and integrated this comparator into the Slot class. Also started a new version of the getWeightedSPARQLQueries().
Modified Paths:
--------------
branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java
branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3.java
branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/sparql/Slot.java
branches/hmm/components-ext/src/test/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3Test.java
Modified: branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java
===================================================================
--- branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java 2012-11-02 17:47:38 UTC (rev 3867)
+++ branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java 2012-11-07 18:13:41 UTC (rev 3868)
@@ -15,6 +15,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -98,7 +99,7 @@
/** The minimum score of items that are accepted from the Sindice search BOA index. **/
private static final Double BOA_THRESHOLD = 0.5;
enum Mode {BEST_QUERY, BEST_NON_EMPTY_QUERY}
- private Mode mode = Mode.BEST_QUERY;
+ private Mode mode = Mode.BEST_NON_EMPTY_QUERY;
/** used to create a label out of the URI when there is no label available in the SPARQL endpoint.*/
private static SimpleIRIShortFormProvider sfp = new SimpleIRIShortFormProvider();
@@ -521,6 +522,7 @@
private SortedSet<WeightedQuery> getWeightedSPARQLQueries(Set<Template> templates, boolean hmm)
{
+// return getWeightedSPARQLQueriesNew(templates);
return hmm?getWeightedSPARQLQueriesWithHMM(templates):getWeightedSPARQLQueriesWithoutHMM(templates);
}
@@ -628,14 +630,42 @@
//
return queries;
}
-
- @SuppressWarnings("unused") private SortedSet<WeightedQuery> getWeightedSPARQLQueriesWithoutHMM(Set<Template> templates){
- logger.debug("Generating SPARQL query candidates...");
-
+
+ /** removes templates which have empty slots */
+ protected Set<Template> goodTemplates(Set<Template> templates)
+ {
+ Set<Template> remainingTemplates = new HashSet<Template>();
+ templates:
+ for(Template t: templates)
+ {
+ for (Slot slot : t.getSlots()) {if(slot.getWords().isEmpty()) {continue templates;} }
+ remainingTemplates.add(t);
+ }
+ return remainingTemplates;
+ }
+
+ /** There seems to be a bug in the other getWeightedSPARQLQueries... functions, so this is a new implementation
+ */
+ protected SortedSet<WeightedQuery> getWeightedSPARQLQueriesNew(Set<Template> templates)
+ {
+ logger.debug("Generating SPARQL query candidates (new implementation)...");
+
+ List<String> vars = new LinkedList<String>();
+ if(templates.isEmpty()) throw new AssertionError("no templates");
+ templates = goodTemplates(templates);
+ if(templates.isEmpty()) throw new AssertionError("no good templates");
+
Map<Slot, Set<Allocation>> slot2Allocations = new TreeMap<Slot, Set<Allocation>>(new Comparator<Slot>() {
@Override
public int compare(Slot o1, Slot o2) {
+ System.err.println(o1.getToken());
+ System.err.println(o2.getToken());
+ if(o1.getToken().equalsIgnoreCase("river")||o2.getToken().equalsIgnoreCase("river"))
+ {
+ int nop = 5;
+ System.err.println(nop);
+ }
if(o1.getSlotType() == o2.getSlotType()){
return o1.getToken().compareTo(o2.getToken());
} else {
@@ -643,18 +673,59 @@
}
}
});
- slot2Allocations = Collections.synchronizedMap(new HashMap<Slot, Set<Allocation>>());
+// slot2Allocations = Collections.synchronizedMap(slot2Allocations);
+ SortedSet<WeightedQuery> allQueries = new TreeSet<WeightedQuery>();
+
+ SortedSet<WeightedQuery> queries = new TreeSet<WeightedQuery>();
+
+ for(Template t : templates)
+ {
+// logger.debug("Processing template:\n" + t.toString());
+ for (Slot slot : t.getSlots())
+ {
+ // get candidates for slot
+ if(!slot2Allocations.containsKey(slot))
+ {
+ slot2Allocations.put(slot,new SlotProcessor(slot).computeAllocations(slot));
+ }
+ }
+ }
+ logger.info(slot2Allocations.size()+" allocations: "+slot2Allocations);
+
+ if(1==1) System.exit(1);
+ return queries;
+ }
+ @SuppressWarnings("unused") private SortedSet<WeightedQuery> getWeightedSPARQLQueriesWithoutHMM(Set<Template> templates){
+ logger.debug("Generating SPARQL query candidates...");
+
+ SortedMap<Slot, Set<Allocation>> slot2Allocations = new TreeMap<Slot, Set<Allocation>>();
+// new Comparator<Slot>() {
+//
+// @Override
+// public int compare(Slot o1, Slot o2) {
+// if(o1.equals(o2)) return 0;
+// return -1;
+//// if(o1.getSlotType() == o2.getSlotType()){
+//// return o1.getToken().compareTo(o2.getToken());
+//// } else {
+//// return -1;
+//// }
+// }
+// });
+ slot2Allocations = Collections.synchronizedSortedMap(slot2Allocations);
+
+
SortedSet<WeightedQuery> allQueries = new TreeSet<WeightedQuery>();
for(Template t : templates)
{
- logger.info("Processing template:\n" + t.toString());
+ logger.debug("Processing template:\n" + t.toString());
// Set<Allocation> allocations = new TreeSet<Allocation>();
boolean containsRegex = t.getQuery().toString().toLowerCase().contains("(regex(");
- ExecutorService executor = Executors.newFixedThreadPool(t.getSlots().size());
+ ExecutorService executor = Executors.newSingleThreadExecutor();//Executors.newFixedThreadPool(t.getSlots().size());
List<Future<Map<Slot, SortedSet<Allocation>>>> list = new ArrayList<Future<Map<Slot, SortedSet<Allocation>>>>();
long startTime = System.currentTimeMillis();
@@ -670,7 +741,8 @@
for (Future<Map<Slot, SortedSet<Allocation>>> future : list) {
try {
Map<Slot, SortedSet<Allocation>> result = future.get();
- Entry<Slot, SortedSet<Allocation>> item = result.entrySet().iterator().next();
+
+ Entry<Slot, SortedSet<Allocation>> item = result.entrySet().iterator().next();
slot2Allocations.put(item.getKey(), item.getValue());
} catch (InterruptedException e) {
e.printStackTrace();
@@ -743,9 +815,22 @@
queries.clear();
queries.addAll(tmp);
tmp.clear();
- }
+ }
- for(Slot slot : sortedSlots){
+ Set<Slot> unhandledSlots = new HashSet<Slot>(sortedSlots);
+ unhandledSlots.removeAll(slot2Allocations.keySet());
+ if(!unhandledSlots.isEmpty())
+ {
+ logger.error("the following slots are unhandled: "+unhandledSlots);
+ }
+ for(Slot slot : sortedSlots)
+ {
+ Set<Allocation> allocations = slot2Allocations.get(slot);
+ if(allocations==null)
+ {
+ System.err.println("no allocations for slot "+slot);
+
+ }
if(!slot2Allocations.get(slot).isEmpty()){
for(Allocation a : slot2Allocations.get(slot)){
for(WeightedQuery query : queries){
@@ -832,9 +917,10 @@
if(!drop){
if(slot.getSlotType() == SlotType.RESOURCE){//avoid queries where predicate is data property and object resource->add REGEX filter in this case
+ q.replaceVarWithURI(slot.getAnchor(), a.getUri());
for(SPARQL_Triple triple : q.getTriplesWithVar(slot.getAnchor())){
SPARQL_Value object = triple.getValue();
-// if(object.isVariable() && object.getName().equals(slot.getAnchor())){//only consider triple where SLOT is in object position
+ if(object.isVariable() && object.getName().equals(slot.getAnchor())){//only consider triple where SLOT is in object position
// SPARQL_Property predicate = triple.getProperty();
// if(!predicate.isVariable()){//only consider triple where predicate is URI
// String predicateURI = predicate.getName().replace("<", "").replace(">", "");
@@ -849,7 +935,7 @@
// }
// } else {
//
-// }
+ }
}
} else {
q.replaceVarWithURI(slot.getAnchor(), a.getUri());
@@ -1306,6 +1392,7 @@
return indexResultItems;
}
+ /** Computes candidates for a slot by using an index. * */
class SlotProcessor implements Callable<Map<Slot, SortedSet<Allocation>>>{
private Slot slot;
@@ -1322,7 +1409,7 @@
}
private SortedSet<Allocation> computeAllocations(Slot slot){
- logger.debug("Computing allocations for slot: " + slot);
+ logger.trace("Computing allocations for slot: " + slot);
SortedSet<Allocation> allocations = new TreeSet<Allocation>();
Index index = getIndexBySlotType(slot);
@@ -1378,7 +1465,7 @@
normProminenceValues(allocations);
computeScore(allocations);
- logger.debug("Found " + allocations.size() + " allocations for slot " + slot);
+ logger.trace(allocations.size() + " allocations for slot " + slot);
return new TreeSet<Allocation>(allocations);
}
Modified: branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3.java
===================================================================
--- branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3.java 2012-11-02 17:47:38 UTC (rev 3867)
+++ branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3.java 2012-11-07 18:13:41 UTC (rev 3868)
@@ -762,7 +762,7 @@
learnedPos++;
List<String> results;
try {
- logger.info("Testing query:\n" + query);
+ logger.debug("Testing query:\n" + query);
com.hp.hpl.jena.query.Query q = QueryFactory.create(query.getQuery().toString(), Syntax.syntaxARQ);
q.setLimit(1);
ResultSet rs = executeSelect(q.toString());
@@ -805,7 +805,7 @@
} else if(queryType == SPARQL_QueryType.ASK){
for(WeightedQuery query : queries){
learnedPos++;
- logger.info("Testing query:\n" + query);
+ logger.debug("Testing query:\n" + query);
boolean result = executeAskQuery(query.getQuery().toString());
learnedSPARQLQueries.add(query);
// if(stopIfQueryResultNotEmpty && result){
Modified: branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/sparql/Slot.java
===================================================================
--- branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/sparql/Slot.java 2012-11-02 17:47:38 UTC (rev 3867)
+++ branches/hmm/components-ext/src/main/java/org/dllearner/algorithm/tbsl/sparql/Slot.java 2012-11-07 18:13:41 UTC (rev 3868)
@@ -5,7 +5,7 @@
import java.util.Iterator;
import java.util.List;
-public class Slot implements Serializable{
+public class Slot implements Serializable, Comparable<Slot> {
private static final long serialVersionUID = 8672756914248710435L;
@@ -164,7 +164,7 @@
if (getClass() != obj.getClass())
return false;
Slot other = (Slot) obj;
- if(other.type == type && other.token == token){
+ if(other.type == type && other.token.equals(token)){
return true;
}
return false;
@@ -178,6 +178,23 @@
result = prime * result + ((type == null) ? 0 : type.hashCode());
return result;
}
+
+ @Override
+ public int compareTo(Slot o)
+ {
+ if(this.equals(o)) return 0;
+ // sort by slot type
+ int t = type.compareTo(o.type);
+ if(t!=0) return t;
+ return token.compareTo(o.token);
+ // sort by
+
+// if(o1.getSlotType() == o2.getSlotType()){
+// return o1.getToken().compareTo(o2.getToken());
+// } else {
+// return -1;
+// }
+ }
}
Modified: branches/hmm/components-ext/src/test/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3Test.java
===================================================================
--- branches/hmm/components-ext/src/test/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3Test.java 2012-11-02 17:47:38 UTC (rev 3867)
+++ branches/hmm/components-ext/src/test/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner3Test.java 2012-11-07 18:13:41 UTC (rev 3868)
@@ -1,6 +1,6 @@
package org.dllearner.algorithm.tbsl.learning;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.File;
@@ -11,7 +11,6 @@
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
@@ -24,6 +23,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,8 +31,10 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
+import java.util.SortedSet;
import java.util.Stack;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -117,9 +119,10 @@
private static final File evaluationFolder = new File("cache/evaluation");
private static final boolean DBPEDIA_PRETAGGED = true;
private static final boolean OXFORD_PRETAGGED = false;
- private static final int MAX_NUMBER_OF_QUESTIONS = 20;
- private static final boolean WHITELIST_ONLY = true;
+ private static final int MAX_NUMBER_OF_QUESTIONS = Integer.MAX_VALUE;
+ private static final boolean WHITELIST_ONLY = false;
private static final Set<Integer> WHITELIST = Collections.unmodifiableSet(new HashSet<Integer>(Arrays.asList(new Integer[] {4})));
+ private static final boolean GENERATE_HTML_ONLY = false;
@Test public void testDBpedia() throws Exception
{
@@ -430,7 +433,7 @@
public void test(String title, final File referenceXML,final SparqlEndpoint endpoint,ExtractionDBCache cache,Knowledgebase kb, Model model, MappingBasedIndex index,boolean pretagged)
throws ParserConfigurationException, SAXException, IOException, TransformerException, ComponentInitException, NoTemplateFoundException
{
- evaluateAndWrite(title,referenceXML,endpoint,cache,kb,model,index,pretagged);
+ if(!GENERATE_HTML_ONLY) {evaluateAndWrite(title,referenceXML,endpoint,cache,kb,model,index,pretagged);}
generateHTML(title);
// if(evaluation.numberOfCorrectAnswers<3) {fail("only " + evaluation.numberOfCorrectAnswers+" correct answers.");}
@@ -529,12 +532,27 @@
{
evaluation.correctlyAnsweredQuestions.add(question);
evaluation.numberOfCorrectAnswers++;
+ evaluation.question2JaccardOfAnswers.put(question,1.0);
}
else
{
- evaluation.incorrectlyAnsweredQuestions.add(question);
- logger.debug("learned queries differing. reference query:\n"+referenceQuery+"\nsuspect query:\n"+suspectQuery);
- logger.debug("learned answers differing: reference answers:\n"+reference.id2Answers.get(i)+"\nsuspect answers:\n"+suspect.id2Answers.get(i));
+ Set<String> intersection = new HashSet<String>(reference.id2Answers.get(i));
+ intersection.retainAll(suspect.id2Answers.get(i));
+ if(!intersection.isEmpty())
+ {
+ evaluation.partlyCorrectlyAnsweredQuestions.add(question);
+ evaluation.numberOfPartlyCorrectAnswers++;
+ Set<String> union = new HashSet<String>(reference.id2Answers.get(i));
+ union.addAll(suspect.id2Answers.get(i));
+ evaluation.question2JaccardOfAnswers.put(question,((double)intersection.size())/union.size());
+ } else
+ {
+ evaluation.incorrectlyAnsweredQuestions.add(question);
+ evaluation.question2JaccardOfAnswers.put(question,0.0);
+ logger.debug("learned queries differing. reference query:\n"+referenceQuery+"\nsuspect query:\n"+suspectQuery);
+ logger.debug("learned answers differing: reference answers:\n"+reference.id2Answers.get(i)+"\nsuspect answers:\n"+suspect.id2Answers.get(i));
+ }
+
}
}
return evaluation;
@@ -542,18 +560,22 @@
static class Evaluation implements Serializable
{
- private static final long serialVersionUID = 5L;
+ private static final long serialVersionUID = 6L;
final QueryTestData testData;
final QueryTestData referenceData;
int numberOfQuestions = 0;
- int numberOfAnsweredQuestions = 0;
+ int numberOfAnsweredQuestions = 0;
int numberOfCorrectAnswers = 0;
+ int numberOfPartlyCorrectAnswers = 0;
double precision = 0;
double recall = 0;
final Set<String> unansweredQuestions = new HashSet<String>();
final Set<String> incorrectlyAnsweredQuestions = new HashSet<String>();
- final Set<String> correctlyAnsweredQuestions = new HashSet<String>();
+ final Set<String> correctlyAnsweredQuestions = new HashSet<String>();
+ final Set<String> partlyCorrectlyAnsweredQuestions = new HashSet<String>();
+ final Map<String,Double> question2JaccardOfAnswers = new HashMap<String,Double>();
+
public Evaluation(QueryTestData testData,QueryTestData referenceData) {this.testData = testData;this.referenceData = referenceData;}
void computePrecisionAndRecall() // we have at maximum one answer set per question
@@ -566,7 +588,7 @@
{
StringBuffer sb = new StringBuffer();
sb.append(numberOfAnsweredQuestions+" of "+numberOfQuestions+" questions answered, ");
- sb.append(numberOfCorrectAnswers+" correct answers.");
+ sb.append(numberOfCorrectAnswers+" exactly correct answers, "+numberOfPartlyCorrectAnswers+" partly correct answers.");
sb.append("precision: "+precision+", recall: "+recall+"\n");
sb.append("Detailed List: ");
sb.append(toHTML());
@@ -578,6 +600,7 @@
StringBuffer sb = new StringBuffer();
sb.append(htmlDetailsList("Unanswered Questions",unansweredQuestions));
sb.append(htmlDetailsList("Wrongly Answered Questions",incorrectlyAnsweredQuestions));
+ sb.append(htmlDetailsList("Partly correctly Answered Questions",partlyCorrectlyAnsweredQuestions));
sb.append(htmlDetailsList("Correctly Answered Questions",correctlyAnsweredQuestions));
return sb.toString();
}
@@ -640,6 +663,13 @@
if (other.correctlyAnsweredQuestions != null) return false;
}
else if (!correctlyAnsweredQuestions.equals(other.correctlyAnsweredQuestions)) return false;
+
+ if (partlyCorrectlyAnsweredQuestions == null)
+ {
+ if (other.partlyCorrectlyAnsweredQuestions != null) return false;
+ }
+ else if (!partlyCorrectlyAnsweredQuestions.equals(other.partlyCorrectlyAnsweredQuestions)) return false;
+
if (incorrectlyAnsweredQuestions == null)
{
if (other.incorrectlyAnsweredQuestions != null) return false;
@@ -1000,14 +1030,14 @@
assertTrue(entities[i][1]+"!="+uri+" "+items,entities[i][1].equals(uri)||entities[i][1].equals(secondUri));
}
}
-
+
/*@Test*/ public void testSolrGoodResults()
{
Knowledgebase dbpedia = createDBpediaLiveKnowledgebase(dbpediaLiveCache);
-
+
testIndex(dbpedia.getResourceIndex(),new String[][]
{{"Brooklyn Bridge","http://dbpedia.org/resource/Brooklyn_Bridge"},{"Estonia","http://dbpedia.org/resource/Estonia"},
- {"Germany","http://dbpedia.org/resource/Germany"}});
+ {"Germany","http://dbpedia.org/resource/Germany"}});
testIndex(dbpedia.getPropertyIndex(),new String[][] {{"born in","http://dbpedia.org/ontology/birthPlace"}});
}
@@ -1282,16 +1312,38 @@
return sbAnswers.toString();
}
- /** Generates the HTML string content for one of the 3 colored bars which represent the correctly, incorrectly and unanswered question.
+ static <K,V extends Comparable<? super V>> SortedSet<Map.Entry<K,V>> entriesSortedByValues(Map<K,V> map) {
+ SortedSet<Map.Entry<K,V>> sortedEntries = new TreeSet<Map.Entry<K,V>>(
+ new Comparator<Map.Entry<K,V>>() {
+ @Override public int compare(Map.Entry<K,V> e1, Map.Entry<K,V> e2) {
+ int res = e1.getValue().compareTo(e2.getValue());
+ return res != 0 ? res : 1; // Special fix to preserve items with equal values
+ }
+ }
+ );
+ sortedEntries.addAll(map.entrySet());
+ return sortedEntries;
+ }
+
+ /** Generates the HTML string content for one of the 4 colored bars which represent the correctly, incorrectly and unanswered question.
* Also creates and links to a file which contains the questions.*/
- private static String createColoredColumn(/*@NonNull*/ File link,/*@NonNull*/ String title,/*@NonNull*/ String color,/*@NonNull*/ Collection<String> questions, int numberOfQuestionsTotal, boolean queriesAvailable, Evaluation evaluation)
+ private static String createColoredColumn(/*@NonNull*/ File link,/*@NonNull*/ String title,/*@NonNull*/ String color,/*@NonNull*/ Collection<String> questions, int numberOfQuestionsTotal, boolean queriesAvailable,boolean jaccard, Evaluation evaluation)
{
final StringBuilder sb = new StringBuilder();
- sb.append("<a href='"+link.getAbsolutePath()+"' title='"+title+"'>");
+ sb.append("<a href='"+link.getAbsolutePath()+"' title='"+title+" ("+questions.size()+"/"+(numberOfQuestionsTotal==0?"":numberOfQuestionsTotal)+")'>");
sb.append("<div style='float:left;width:"+100.0*questions.size()/numberOfQuestionsTotal+"%;height:1em;background-color:"+color+";'></div>");
sb.append("</a>");
+ // link.getParentFile().mkdirs();
+ Collection<String> sortedQuestions;
+ if(jaccard) // sort by jaccard descending
+ {
+ sortedQuestions = new LinkedList<String>();
+ SortedMap<String,Double> map = new TreeMap<String,Double>();
+ for(String question : questions) {map.put(question, 1-evaluation.question2JaccardOfAnswers.get(question));}
- // link.getParentFile().mkdirs();
+ for(Entry<String,Double> e: entriesSortedByValues(map)) {sortedQuestions.add(e.getKey());}
+ } else sortedQuestions = questions;
+
try
{
PrintWriter out = new PrintWriter(link);
@@ -1301,8 +1353,9 @@
out.println("<!DOCTYPE html><html>\n<head><title>"+title+"</title></head>\n<body>\n<table border='1'>");
if(queriesAvailable)
{
- out.println("<tr><th>Question</th><th>Learned Query</th><th>Reference Query</th><th>Learned Answers</th><th>Reference Answers</th><th>Error Type</th></tr>");
- for(String question: questions)
+ out.println("<tr><th>Question</th><th>Learned Query</th><th>Reference Query</th><th>Learned Answers</th><th>Reference Answers</th><th>Error Type</th>"+
+ (jaccard?"<th>jaccard</th>":"")+"</tr>");
+ for(String question: sortedQuestions)
{
Integer id = question2Id.get(question);
if(evaluation.testData.id2Answers.get(id)==null) {System.err.println(question);continue;}
@@ -1312,12 +1365,13 @@
"<td><code><pre>"+escapePre(evaluation.referenceData.id2Query.get(id))+"</pre></code></td>"+
"<td><ul>"+getAnswerHTMLList(evaluation.testData.id2Answers.get(id).toArray(new String[0]))+"</ul></td>"+
"<td><ul>"+getAnswerHTMLList(evaluation.referenceData.id2Answers.get(id).toArray(new String[0]))+"</ul></td>"+
- "<td>"+evaluation.testData.id2LearnStatus.get(id)+"</td></tr>");
+ "<td>"+evaluation.testData.id2LearnStatus.get(id)+"</td>"+
+ "<td>"+(jaccard?evaluation.question2JaccardOfAnswers.get(question):"")+"</td></tr>");
}
} else
- {
+ {
out.println("<tr><th>Question</th><th>Error Type</th></tr>");
- for(String question: questions)
+ for(String question: sortedQuestions)
{
Integer id = question2Id.get(question);
if(id==null) {System.err.println(question);continue;}
@@ -1350,6 +1404,7 @@
out.println("</style></head>");
out.println("<body>");
out.println(diffHTML("Correctly Answered Questions (precision and recall = 1)", from.correctlyAnsweredQuestions, to.correctlyAnsweredQuestions));
+ out.println(diffHTML("Partly correctly Answered Questions", from.partlyCorrectlyAnsweredQuestions, to.partlyCorrectlyAnsweredQuestions));
out.println(diffHTML("Incorrectly Answered Questions", from.incorrectlyAnsweredQuestions, to.incorrectlyAnsweredQuestions));
out.println(diffHTML("Unanswered Questions", from.unansweredQuestions, to.unansweredQuestions));
out.println("</body>\n</html>");
@@ -1390,9 +1445,10 @@
}
sb2.append("</td><td width='100%'>");
sb2.append("<div style='width:100%;height:1em;border:solid 1px;'>");
- sb2.append(createColoredColumn(new File(folder,"correctly_answered.html"), "Correctly Answered Questions", "green", e.correctlyAnsweredQuestions, e.numberOfQuestions,true,e));
- sb2.append(createColoredColumn(new File(folder,"incorrectly_answered.html"), "Incorrectly Answered Questions", "orange", e.incorrectlyAnsweredQuestions, e.numberOfQuestions,true,e));
- sb2.append(createColoredColumn(new File(folder,"unanswered.html"), "Unanswered Questions", "red", e.unansweredQuestions, e.numberOfQuestions,false,e));
+ sb2.append(createColoredColumn(new File(folder,"correctly_answered.html"), "Correctly Answered Questions", "green", e.correctlyAnsweredQuestions, e.numberOfQuestions,true,false,e));
+ sb2.append(createColoredColumn(new File(folder,"partly_correctly_answered.html"), "Partly Correctly Answered Questions", "gold", e.partlyCorrectlyAnsweredQuestions, e.numberOfQuestions,true,true,e));
+ sb2.append(createColoredColumn(new File(folder,"incorrectly_answered.html"), "Incorrectly Answered Questions", "darkorange", e.incorrectlyAnsweredQuestions, e.numberOfQuestions,true,false,e));
+ sb2.append(createColoredColumn(new File(folder,"unanswered.html"), "Unanswered Questions", "red", e.unansweredQuestions, e.numberOfQuestions,false,false,e));
sb2.append("<span style='width:1000px;'></span>");
sb2.append("</td></tr>\n");
last = e;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
