Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[3747] trunk/components-ext/src
|
From: <lor...@us...> - 2012-06-15 14:01:27
|
Revision: 3747
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=3747&view=rev
Author: lorenz_b
Date: 2012-06-15 14:01:20 +0000 (Fri, 15 Jun 2012)
Log Message:
-----------
Updates for oxford eval.
Modified Paths:
--------------
trunk/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java
trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/OxfordEvaluation.java
trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/TBSLTest.java
Added Paths:
-----------
trunk/components-ext/src/main/resources/tbsl/oxford_eval_additional_queries.txt
trunk/components-ext/src/main/resources/tbsl/oxford_eval_queries.txt
Removed Paths:
-------------
trunk/components-ext/src/main/resources/tbsl/evaluation.txt
Modified: trunk/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java
===================================================================
--- trunk/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java 2012-06-14 18:41:30 UTC (rev 3746)
+++ trunk/components-ext/src/main/java/org/dllearner/algorithm/tbsl/learning/SPARQLTemplateBasedLearner2.java 2012-06-15 14:01:20 UTC (rev 3747)
@@ -123,10 +123,12 @@
private Map<Template, Collection<? extends Query>> template2Queries;
private Map<Slot, List<String>> slot2URI;
- private Set<WeightedQuery> generatedQueries;
+ private SortedSet<WeightedQuery> generatedQueries;
private SPARQLReasoner reasoner;
+ private String currentlyExecutedQuery;
+
public SPARQLTemplateBasedLearner2(SparqlEndpoint endpoint, Index resourcesIndex, Index classesIndex, Index propertiesIndex){
this(endpoint, resourcesIndex, classesIndex, propertiesIndex, new StanfordPartOfSpeechTagger());
}
@@ -277,6 +279,7 @@
learnedSPARQLQueries = new HashMap<String, Object>();
template2Queries = new HashMap<Template, Collection<? extends Query>>();
slot2URI = new HashMap<Slot, List<String>>();
+ currentlyExecutedQuery = null;
}
public void learnSPARQLQueries() throws NoTemplateFoundException{
@@ -317,12 +320,12 @@
}
- public Set<WeightedQuery> getGeneratedQueries() {
+ public SortedSet<WeightedQuery> getGeneratedQueries() {
return generatedQueries;
}
- public Set<WeightedQuery> getGeneratedQueries(int topN) {
- Set<WeightedQuery> topNQueries = new TreeSet<WeightedQuery>();
+ public SortedSet<WeightedQuery> getGeneratedQueries(int topN) {
+ SortedSet<WeightedQuery> topNQueries = new TreeSet<WeightedQuery>();
int max = Math.min(topN, generatedQueries.size());
for(WeightedQuery wQ : generatedQueries){
topNQueries.add(wQ);
@@ -382,7 +385,7 @@
}
- private Set<WeightedQuery> getWeightedSPARQLQueries(Set<Template> templates){
+ private SortedSet<WeightedQuery> getWeightedSPARQLQueries(Set<Template> templates){
logger.info("Generating SPARQL query candidates...");
Map<Slot, Set<Allocation>> slot2Allocations = new TreeMap<Slot, Set<Allocation>>(new Comparator<Slot>() {
@@ -399,7 +402,7 @@
slot2Allocations = Collections.synchronizedMap(new HashMap<Slot, Set<Allocation>>());
- Set<WeightedQuery> allQueries = new TreeSet<WeightedQuery>();
+ SortedSet<WeightedQuery> allQueries = new TreeSet<WeightedQuery>();
Set<Allocation> allocations;
@@ -864,6 +867,7 @@
}
private boolean executeAskQuery(String query){
+ currentlyExecutedQuery = query;
QueryEngineHTTP qe = new QueryEngineHTTP(endpoint.getURL().toString(), query);
for(String uri : endpoint.getDefaultGraphURIs()){
qe.addDefaultGraph(uri);
@@ -873,6 +877,7 @@
}
private ResultSet executeSelect(String query) {
+ currentlyExecutedQuery = query;
ResultSet rs;
if (model == null) {
if (cache == null) {
@@ -890,6 +895,9 @@
return rs;
}
+ public String getCurrentlyExecutedQuery() {
+ return currentlyExecutedQuery;
+ }
public int getLearnedPosition() {
if(learnedPos >= 0){
Deleted: trunk/components-ext/src/main/resources/tbsl/evaluation.txt
===================================================================
--- trunk/components-ext/src/main/resources/tbsl/evaluation.txt 2012-06-14 18:41:30 UTC (rev 3746)
+++ trunk/components-ext/src/main/resources/tbsl/evaluation.txt 2012-06-15 14:01:20 UTC (rev 3747)
@@ -1,149 +0,0 @@
-question: houses in Headington
-
-question: houses in Abingdon with more than 2 bedrooms
-
-question: houses with garden in Wheatley
-
-question: detached houses in Oxford
-
-question: Victorian houses in Oxfordshire
-
-question: Edwardian house in Oxfordshire for less than 1000000
-
-question: houses with double garage
-
-question: houses with large garden and equipped kitchen
-
-question: houses with more than 1 reception room
-
-question: houses in Didcot furnished to a high standard
-
-question: houses with conservatory room and less than 900000 pounds
-
-question: detached bungalows in Oxfordshire
-
-question: houses in Old Marston
-
-question: family houses with more than 2 bathrooms and more than 4 bedrooms
-
-question: houses close to Iffley Sport Centre
-
-question: houses in Oxford close to the train station
-
-question: houses in Summertown for less than 400000 pounds
-
-question: two floors houses in East Oxford
-
-question: brand new houses in Oxford for less than 500000 pounds
-
-question: houses close to Brookes University
-
-question: houses in Jericho area
-
-question: house close to Headington hospitals
-
-question: modern houses with gas central heating
-
-question: houses with electric heating
-
-question: houses less than 500000 within area OX16
-
-question: houses close to an Italian restaurant
-
-question: houses at walking distance from a pharmacy
-
-question: houses at walking distance from Tesco or Sainsburys shops
-
-question: houses nearby Sheldonian Theatre
-
-question: houses with underfloor heating
-
-question: houses with wood floor
-
-question: houses close to The King's Arms pub
-
-question: houses with garden large at least 2 acres
-
-question: houses with many reception rooms
-
-question: houses built around 1950
-
-question: houses with balcony
-
-question: houses with double glazed windows
-
-question: houses far from city centre
-
-question: 2 bedroom houses near oxford train station
-
-question: 4 bedroom detached houses in oxford
-
-question: studio apartments in summertown, Oxford
-
-question: freehold houses with 2 bedrooms and a living room in banbury
-
-question: houses in Oxford city centre with at most 2 bedrooms
-
-question: houses with garage within minutes of Oxford schools and in a quiet road
-
-question: victorian town houses in north Oxford
-
-question: terrace houses with west facing garden
-
-question: modernised end terrace houses with private parking
-
-question: three bedroom houses with open fireplace
-
-question: houses available from June 15th.
-
-question: houses on rawlinson road
-
-question: flats near supermarket
-
-question: flats with bill included
-
-
-
-question: give me flats in central Oxford with at least one bedroom below 1000 GBP a month?
-
-question: Give me all 2 bedroom flats in walking distance from the computer science departement!
-
-question: Give me all houses with 3 bedrooms or more, close to the train station with good shopping opportunities.
-
-question: find a property with 2 bedrooms close to some park.
-
-question: Give me all flats at roughly 1300 GBP the month, equally close to the computer science department and Christ Church College.
-
-question: Give me all flats in the area around Cowley Road with 2 bedrooms.
-
-question: Give me all furnished places with one bedroom close to the Radcliffe Camera.
-
-question: Give me all unfurnished houses with at least 2 bedrooms in Summertown.
-
-question: Give me all furnished flats with one bedroom for smokers.
-
-question: Give me all flats with parking in central Oxford.
-
-question: Give me all cheap places in Cowley.
-
-question: Give me all representative houses in Summertown.
-
-question: find a property for sale, with 2 bedrooms, parking, close to shops.
-
-question: Give me all flats with a garden, one bedroom, in walking distance to the computer science departement
-
-question: Give me all places offered close to the train station?
-
-question: Give me all retirement houses for sale near Oxford.
-
-question: Give me all houses that I can BBQ.
-
-question: Give me all flats which are far from the river.
-
-question: Give me all flats which are close to three bars.
-
-question: What is the average price of furnished 1 bedroom apartments in Heddington?
-
-question: How many flats are offered in central Oxford below 1000 GBP a month?
-
-question: Which area in Oxford has the cheapest offers close to the Keble college [or some other landmark]?
Added: trunk/components-ext/src/main/resources/tbsl/oxford_eval_additional_queries.txt
===================================================================
--- trunk/components-ext/src/main/resources/tbsl/oxford_eval_additional_queries.txt (rev 0)
+++ trunk/components-ext/src/main/resources/tbsl/oxford_eval_additional_queries.txt 2012-06-15 14:01:20 UTC (rev 3747)
@@ -0,0 +1,79 @@
+question: Give me all houses close to The Ashmolean Museum, with more than one bedroom
+
+question: Give me all houses in Botley Road
+
+question: Give me all houses in Littlemore
+
+question: Give me all houses that cost between 600000 and 800000 pounds
+
+question: Give me all houses with 3 bedrooms in Florence Park Road
+
+question: Give me all houses with front garden and rear garden
+
+//question: Give me all houses with 2 bathrooms, 5 bedrooms and at least 1 reception
+
+question: Give me all houses in Oxford, close to Railway Station
+
+question: Give me all houses with ample parking
+
+question: Give me all house with electric central heating system
+
+question: Give me all houses with three en-suites near Oxford
+
+question: Give me all houses with countryside views
+
+question: Give me all houses with farmland views
+
+question: Give me all houses nearby River Thames
+
+question: Give me all houses having one utility room or cloakroom
+
+question: Give me all houses in Oxfordshire with fireplaces
+
+question: Give me all houses with open plan kitchen near Oxford
+
+question: Give me all houses with walled garden near Oxford
+
+question: Give me all houses with river views
+
+question: Give me all houses with gated entrance or in a gated community
+
+question: Give me all recently refurbished houses with lift (access)
+
+question: Give me all houses with balcony and vaulted ceiling
+
+question: Give me all houses that are a maisonette and on Gloucester Green
+
+question: Give me all houses in a corner or end-of-terrace plot
+
+question: Give me all houses with 3 bedrooms, but cheaper than 150000 GBP
+
+question: Give me all houses with at least 2 reception rooms and a garden
+
+question: Give me all houses in/with a courtyard
+
+question: Give me all houses with fireplace and a garden that is not communal
+
+question: Give me all houses with parking but not Victorian
+
+//question: Give me all houses with 2-5 bedrooms, a balcony, and a price less than 450000 GBP
+
+question: Give me all houses with 3 bathrooms, but not en-suite
+
+question: Give me all houses in a retirement complex
+
+question: Give me all houses with double glazing and central heating
+
+question: Give me all houses with “no upper chain” (means immediately available)
+
+question: Give me all houses listed Grade I or Grade II (means “under protection for age …”)
+
+question: Give me all houses that have recently replumbed
+
+question: Give me all houses in Witney or Wolvercote
+
+question: Give me all houses with more than 2 bedrooms, but not in Marston
+
+question: Give me all houses in Banbury Road
+
+question: Give me all houses in the area of (John) Radcliffe hospital
\ No newline at end of file
Copied: trunk/components-ext/src/main/resources/tbsl/oxford_eval_queries.txt (from rev 3746, trunk/components-ext/src/main/resources/tbsl/evaluation.txt)
===================================================================
--- trunk/components-ext/src/main/resources/tbsl/oxford_eval_queries.txt (rev 0)
+++ trunk/components-ext/src/main/resources/tbsl/oxford_eval_queries.txt 2012-06-15 14:01:20 UTC (rev 3747)
@@ -0,0 +1,149 @@
+question: houses in Headington
+
+question: houses in Abingdon with more than 2 bedrooms
+
+question: houses with garden in Wheatley
+
+question: detached houses in Oxford
+
+question: Victorian houses in Oxfordshire
+
+question: Edwardian house in Oxfordshire for less than 1000000
+
+question: houses with double garage
+
+question: houses with large garden and equipped kitchen
+
+question: houses with more than 1 reception room
+
+question: houses in Didcot furnished to a high standard
+
+question: houses with conservatory room and less than 900000 pounds
+
+question: detached bungalows in Oxfordshire
+
+question: houses in Old Marston
+
+question: family houses with more than 2 bathrooms and more than 4 bedrooms
+
+question: houses close to Iffley Sport Centre
+
+question: houses in Oxford close to the train station
+
+question: houses in Summertown for less than 400000 pounds
+
+question: two floors houses in East Oxford
+
+question: brand new houses in Oxford for less than 500000 pounds
+
+question: houses close to Brookes University
+
+question: houses in Jericho area
+
+question: house close to Headington hospitals
+
+question: modern houses with gas central heating
+
+question: houses with electric heating
+
+question: houses less than 500000 within area OX16
+
+question: houses close to an Italian restaurant
+
+question: houses at walking distance from a pharmacy
+
+question: houses at walking distance from Tesco or Sainsburys shops
+
+question: houses nearby Sheldonian Theatre
+
+question: houses with underfloor heating
+
+question: houses with wood floor
+
+question: houses close to The King's Arms pub
+
+question: houses with garden large at least 2 acres
+
+question: houses with many reception rooms
+
+question: houses built around 1950
+
+question: houses with balcony
+
+question: houses with double glazed windows
+
+question: houses far from city centre
+
+question: 2 bedroom houses near oxford train station
+
+question: 4 bedroom detached houses in oxford
+
+question: studio apartments in summertown, Oxford
+
+question: freehold houses with 2 bedrooms and a living room in banbury
+
+question: houses in Oxford city centre with at most 2 bedrooms
+
+question: houses with garage within minutes of Oxford schools and in a quiet road
+
+question: victorian town houses in north Oxford
+
+question: terrace houses with west facing garden
+
+question: modernised end terrace houses with private parking
+
+question: three bedroom houses with open fireplace
+
+question: houses available from June 15th.
+
+question: houses on rawlinson road
+
+question: flats near supermarket
+
+question: flats with bill included
+
+
+
+question: give me all flats in central Oxford with at least one bedroom below 1000 GBP a month.
+
+question: Give me all 2 bedroom flats in walking distance from the computer science departement!
+
+//question: Give me all houses with 3 bedrooms or more, close to the train station with good shopping opportunities.
+
+question: find a property with 2 bedrooms close to some park.
+
+//question: Give me all flats at roughly 1300 GBP the month, equally close to the computer science department and Christ Church College.
+
+question: Give me all flats in the area around Cowley Road with 2 bedrooms.
+
+question: Give me all furnished places with one bedroom close to the Radcliffe Camera.
+
+question: Give me all unfurnished houses with at least 2 bedrooms in Summertown.
+
+question: Give me all furnished flats with one bedroom for smokers.
+
+question: Give me all flats with parking in central Oxford.
+
+question: Give me all cheap places in Cowley.
+
+question: Give me all representative houses in Summertown.
+
+//question: find a property for sale, with 2 bedrooms, parking, close to shops.
+
+//question: Give me all flats with a garden, one bedroom, in walking distance to the computer science departement
+
+question: Give me all places offered close to the train station?
+
+question: Give me all retirement houses for sale near Oxford.
+
+question: Give me all houses that I can BBQ.
+
+question: Give me all flats which are far from the river.
+
+question: Give me all flats which are close to three bars.
+
+question: What is the average price of furnished 1 bedroom apartments in Heddington?
+
+question: How many flats are offered in central Oxford below 1000 GBP a month?
+
+question: Which area in Oxford has the cheapest offers close to the Keble college [or some other landmark]?
Modified: trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/OxfordEvaluation.java
===================================================================
--- trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/OxfordEvaluation.java 2012-06-14 18:41:30 UTC (rev 3746)
+++ trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/OxfordEvaluation.java 2012-06-15 14:01:20 UTC (rev 3747)
@@ -10,6 +10,7 @@
import java.util.HashMap;
import java.util.Map;
+import org.dllearner.algorithm.tbsl.learning.NoTemplateFoundException;
import org.dllearner.algorithm.tbsl.learning.SPARQLTemplateBasedLearner2;
import org.dllearner.common.index.MappingBasedIndex;
import org.dllearner.common.index.SPARQLIndex;
@@ -19,13 +20,20 @@
import org.dllearner.kb.sparql.ExtractionDBCache;
import org.dllearner.kb.sparql.SparqlEndpoint;
+import com.hp.hpl.jena.query.QueryParseException;
+import com.hp.hpl.jena.sparql.engine.http.QueryExceptionHTTP;
+
public class OxfordEvaluation {
- private static final String QUERIES_FILE = OxfordEvaluation.class.getClassLoader().getResource("tbsl/evaluation.txt").getPath();
+ private static final String QUERIES_FILE1 = OxfordEvaluation.class.getClassLoader().getResource("tbsl/oxford_eval_queries.txt").getPath();
+ private static final String QUERIES_FILE2 = OxfordEvaluation.class.getClassLoader().getResource("tbsl/oxford_eval_additional_queries.txt").getPath();
+ private static final String LOG_DIRECTORY = "log/oxford/";
+ private static final String LOG_FILE = "evaluation.txt";
public static void main(String[] args) throws Exception{
SparqlEndpoint endpoint = new SparqlEndpoint(new URL("http://lgd.aksw.org:8900/sparql"), Collections.singletonList("http://diadem.cs.ox.ac.uk"), Collections.<String>emptyList());
ExtractionDBCache cache = new ExtractionDBCache("cache");
+ new File(LOG_DIRECTORY).mkdirs();
SPARQLIndex resourcesIndex = new VirtuosoResourcesIndex(endpoint, cache);
SPARQLIndex classesIndex = new VirtuosoClassesIndex(endpoint, cache);
@@ -44,45 +52,69 @@
int learnedQuestions = 0;
Map<String, String> question2QueryMap = new HashMap<String, String>();
- BufferedReader in = new BufferedReader(new FileReader(new File(QUERIES_FILE)));
- BufferedWriter out = new BufferedWriter(new FileWriter(new File("log/oxford_eval.txt")));
+ BufferedReader in = new BufferedReader(new FileReader(new File(QUERIES_FILE2)));
+ BufferedWriter out = new BufferedWriter(new FileWriter(new File(LOG_DIRECTORY + LOG_FILE), false));
- int questionNr = 0;
+ int questionCnt = 0;
int errorCnt = 0;
- int noQueryCnt = 0;
+ int noTemplateFoundCnt = 0;
+ int noQueryWithNonEmptyResultSetCnt = 0;
String question = null;
while((question = in.readLine()) != null){
question = question.replace("question:", "").trim();
- if(question.isEmpty()) continue;
+ if(question.isEmpty() || question.startsWith("//")) continue;
if(!question.toLowerCase().contains("Give me all") && Character.isLowerCase(question.charAt(0))){
question = "Give me all " + question;
}
System.out.println("########################################################");
- questionNr++;
+ questionCnt++;
System.out.println(question);
try {
+ out.write("****************************************\n");
+ out.write("QUESTION: " + question + "\n");
learner.setQuestion(question);
learner.learnSPARQLQueries();
String learnedQuery = learner.getBestSPARQLQuery();
if(learnedQuery != null){
question2QueryMap.put(question, learnedQuery);
learnedQuestions++;
- out.write("****************************************\n" + question + "\n" + learnedQuery + "\n****************************************");
+ out.write("ANSWER FOUND: YES\n");
+ out.write(learnedQuery + "\n");
} else {
- noQueryCnt++;
- out.write("****************************************\n" + question + "\nNO QUERY WITH NON-EMPTY RESULTSET FOUND\n****************************************");
+ noQueryWithNonEmptyResultSetCnt++;
+ out.write("ANSWER FOUND: NO\n");
+ out.write("REASON: NO SPARQL QUERY WITH NON-EMPTY RESULTSET FOUND\n");
+ out.write("SPARQL QUERY WITH HIGHEST SCORE TESTED:\n" + learner.getGeneratedQueries().first());
+
}
} catch (Exception e) {
e.printStackTrace();
+ out.write("ANSWER FOUND: NO\n");
+ if(e instanceof NoTemplateFoundException){
+ noTemplateFoundCnt++;
+ out.write("REASON: NO TEMPLATE FOUND");
+ } else {
+ errorCnt++;
+ out.write("REASON: ERROR OCCURED (" + e.getClass() + ")\n");
+ if(e instanceof QueryExceptionHTTP || e instanceof QueryParseException){
+ out.write("\nLast tested SPARQL query: " + learner.getCurrentlyExecutedQuery());
+ }
+ }
+ } catch (Error e){
+ e.printStackTrace();
+ out.write("ANSWER FOUND: NO\n");
errorCnt++;
- out.write("****************************************\n" + question + "\nERROR: " + e.getClass() + "\n****************************************");
+ out.write("REASON: ERROR OCCURED (" + e.getClass() + ")\n");
}
+ out.write("\n****************************************");
out.flush();
}
- out.write("################################\n");
- out.write("Questions with answer: " + learnedQuestions + "\n");
- out.write("Questions with no answer (and no error): " + noQueryCnt + "\n");
- out.write("Questions with error: " + errorCnt + "\n");
+ out.write("\n\n###################SUMMARY################\n");
+ out.write("Questions tested:\t" + questionCnt + "\n");
+ out.write("Questions with answer:\t" + learnedQuestions + "\n");
+ out.write("Questions with no answer (and no error):\t" + noQueryWithNonEmptyResultSetCnt + "\n");
+ out.write("Questions with no templates:\t" + noTemplateFoundCnt + "\n");
+ out.write("Questions with other errors:\t" + errorCnt + "\n");
in.close();
out.close();
Modified: trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/TBSLTest.java
===================================================================
--- trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/TBSLTest.java 2012-06-14 18:41:30 UTC (rev 3746)
+++ trunk/components-ext/src/test/java/org/dllearner/algorithm/tbsl/TBSLTest.java 2012-06-15 14:01:20 UTC (rev 3747)
@@ -118,6 +118,8 @@
String question = "Give me all houses near a school.";
question = "Give me all houses with more than 3 bathrooms and more than 2 bedrooms.";
question = "Give me all Victorian houses in Oxfordshire";
+ question = "Give me all Edwardian house in Oxfordshire for less than 1000000.";
+// question = "Give me all family houses with more than 2 bathrooms and more than 4 bedrooms";
learner.setQuestion(question);
learner.learnSPARQLQueries();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
