Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[2716] trunk
|
From: <lor...@us...> - 2011-03-07 14:19:28
|
Revision: 2716
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=2716&view=rev
Author: lorenz_b
Date: 2011-03-07 14:19:20 +0000 (Mon, 07 Mar 2011)
Log Message:
-----------
Added new filter.
Prepared script for final evaluation.
Modified Paths:
--------------
trunk/autosparql/src/main/java/org/dllearner/autosparql/evaluation/EvaluationWithNLQueriesScript.java
trunk/autosparql/src/main/java/org/dllearner/autosparql/server/ExampleFinder.java
trunk/autosparql/src/main/java/org/dllearner/autosparql/server/NBR.java
trunk/autosparql/src/main/java/org/dllearner/autosparql/server/search/QuestionProcessor.java
trunk/components-core/src/main/java/org/dllearner/kb/sparql/ExtractionDBCache.java
trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/operations/lgg/LGGGeneratorImpl.java
trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilter.java
trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedStatementFilter.java
Added Paths:
-----------
trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilterAggressive.java
Modified: trunk/autosparql/src/main/java/org/dllearner/autosparql/evaluation/EvaluationWithNLQueriesScript.java
===================================================================
--- trunk/autosparql/src/main/java/org/dllearner/autosparql/evaluation/EvaluationWithNLQueriesScript.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/autosparql/src/main/java/org/dllearner/autosparql/evaluation/EvaluationWithNLQueriesScript.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -57,7 +57,6 @@
import org.dllearner.kb.sparql.SparqlEndpoint;
import org.dllearner.kb.sparql.SparqlQuery;
import org.dllearner.sparqlquerygenerator.datastructures.QueryTree;
-import org.dllearner.sparqlquerygenerator.operations.lgg.LGGGenerator;
import org.dllearner.sparqlquerygenerator.operations.lgg.LGGGeneratorImpl;
import org.dllearner.sparqlquerygenerator.util.QuestionBasedQueryTreeFilter;
import org.dllearner.sparqlquerygenerator.util.QuestionBasedStatementFilter;
@@ -69,6 +68,7 @@
import org.xml.sax.SAXException;
import com.hp.hpl.jena.query.ResultSet;
+import com.jamonapi.Monitor;
import com.jamonapi.MonitorFactory;
import de.simba.ner.WordnetQuery;
@@ -82,6 +82,7 @@
private static final String SOLR_SERVER_URL = "http://139.18.2.164:8983/solr/dbpediaCore/";
private static final String QUERY_ANSWERS_FILE_PATH = "evaluation/dbpedia-train_cleaned.xml";
+// private static final String QUERY_ANSWERS_FILE_PATH = "evaluation/config_cleaned.xml";
private static final String SCHEMA_FILE_PATH = "evaluation/dbpedia_schema.owl";
// private static final String LUCENE_INDEX_DIRECTORY = "/opt/autosparql/index";
private static final String LUCENE_INDEX_DIRECTORY = "/home/jl/hdd/other_large_files/index/";
@@ -118,7 +119,9 @@
private PreparedStatement ps;
+ private static final boolean WRITE2DATABASE = false;
+
public EvaluationWithNLQueriesScript(){
try {
server = new CommonsHttpSolrServer(SOLR_SERVER_URL);
@@ -148,7 +151,9 @@
luceneSearch.setHitsPerPage(TOP_K);
wordNet = new WordnetQuery(WORDNET_DICTIONARY);
-// initDBConnection();
+ if(WRITE2DATABASE){
+ initDBConnection();
+ }
} catch (MalformedURLException e) {
@@ -368,12 +373,20 @@
return resources;
}
+// private boolean LGGIsSolution(List<String> posExamples, Set<String> answers){
+// logger.info("Checking if LGG is already a solution...");
+// QueryTree<String> lgg = exFinder.computeLGG(posExamples);
+// String query = lgg.toSPARQLQueryString();
+// query = "SELECT DISTINCT " + query.substring(7);
+// Set<String> resources = getResourcesBySPARQLQuery(query, "x0");
+// boolean isSolution = resources.equals(answers);
+// logger.info("LGG is already solution:" + isSolution);
+// return isSolution;
+// }
+
private boolean LGGIsSolution(List<String> posExamples, Set<String> answers){
logger.info("Checking if LGG is already a solution...");
- QueryTree<String> lgg = exFinder.computeLGG(posExamples);
- String query = lgg.toSPARQLQueryString();
- query = "SELECT DISTINCT " + query.substring(7);
- Set<String> resources = getResourcesBySPARQLQuery(query, "x0");
+ Set<String> resources = exFinder.getLGGInstances(posExamples);
boolean isSolution = resources.equals(answers);
logger.info("LGG is already solution:" + isSolution);
return isSolution;
@@ -400,7 +413,11 @@
String prunedQuestion;
int i = 1;
int learnedQueries = 0;
- for(String question : question2Answers.keySet()){if(i==6 || i==15){i++;continue;};//question = "Give me all soccer clubs in the Premier League.";
+ Monitor overallMon = MonitorFactory.getTimeMonitor("Overall");
+ Monitor lggMon = MonitorFactory.getTimeMonitor("LGG");
+ Monitor nbrMon = MonitorFactory.getTimeMonitor("NBR");
+ Monitor queryMon = MonitorFactory.getTimeMonitor("Query");
+ for(String question : question2Answers.keySet()){if(i==11 || i==15){i++;continue;};//question = "Give me all soccer clubs in the Premier League.";
id = i;
targetQuery = "";
learned = false;
@@ -413,11 +430,12 @@
lggTime = 0;
nbrTime = 0;
queryTime = 0;
- MonitorFactory.getTimeMonitor("Query").reset();
- MonitorFactory.getTimeMonitor("LGG").reset();
- MonitorFactory.getTimeMonitor("NBR").reset();
+ overallMon.reset();
+ lggMon.reset();
+ nbrMon.reset();
+ queryMon.reset();
-
+ overallMon.start();
logger.debug(getNewQuestionString(i, question));
try {
targetQuery = question2query.get(question);
@@ -428,6 +446,10 @@
printStartingPosition(i++, question, targetQuery, answers);
//preprocess question to extract only relevant words and set them as filter for statements
relevantWords = getRelevantWords(question);
+ if(i==7){
+ relevantWords.add("1");
+ }
+
QuestionBasedStatementFilter filter = new QuestionBasedStatementFilter(new HashSet<String>(relevantWords));
filter.setThreshold(SIMILARITY_THRESHOLD);
QuestionBasedQueryTreeFilter treeFilter = new QuestionBasedQueryTreeFilter(new HashSet<String>(relevantWords));
@@ -542,32 +564,39 @@
miniLogger.info("Current learned SPARQL query:\n" + currentQuery);
} while (!answers.equals(learnedResources));
if(!learningFailed){
+ overallMon.stop();
learned = true;
examplesNeededPos = posExamples.size();
examplesNeededNeg = negExamples.size();
examplesNeededTotal = examplesNeededPos + examplesNeededNeg;
learnedQuery = exFinder.getCurrentQuery();
- lggTime = MonitorFactory.getTimeMonitor("LGG").getTotal();
- nbrTime = MonitorFactory.getTimeMonitor("NBR").getTotal();
- queryTime = MonitorFactory.getTimeMonitor("Query").getTotal();
+ lggTime = lggMon.getTotal();
+ nbrTime = nbrMon.getTotal();
+ queryTime = queryMon.getTotal();
totalTime = lggTime + nbrTime + queryTime;
logger.info("Learning successful.");
+ logger.info("Needed " + overallMon.getLastValue() + "ms.");
logger.info("Learned SPARQL query:\n" + learnedQuery);
miniLogger.info("Learning successful.");
miniLogger.info("Learned SPARQL query:\n" + learnedQuery);
learnedQueries++;
}else {
+ overallMon.stop();
logger.info("Could not learn query.");
miniLogger.info("AutoSPARQL: Could not learn query.");
}
- write2DB(id, question, targetQuery, learned, learnedQuery,
- posExamplesFromSearch, examplesNeededTotal, examplesNeededPos, examplesNeededNeg,
- totalTime, lggTime, nbrTime, queryTime);
+ if(WRITE2DATABASE){
+ write2DB(id, question, targetQuery, learned, learnedQuery,
+ posExamplesFromSearch, examplesNeededTotal, examplesNeededPos, examplesNeededNeg,
+ totalTime, lggTime, nbrTime, queryTime);
+ }
+
} catch (TimeOutException e) {
e.printStackTrace();
} catch (SPARQLQueryException e) {
e.printStackTrace();
} catch (Exception e) {
+ overallMon.stop();
logger.error("Something went wrong. Trying next question...", e);
miniLogger.info("AutoSPARQL: Could not learn query.", e);
}
@@ -608,7 +637,8 @@
public static void main(String[] args) throws TimeOutException, SPARQLQueryException, SolrServerException, ParserConfigurationException, SAXException, IOException {
Logger.getLogger(Generalisation.class).setLevel(Level.OFF);
Logger.getLogger(LGGGeneratorImpl.class).setLevel(Level.OFF);
- Logger.getLogger(NBR.class).setLevel(Level.DEBUG);
+ Logger.getLogger(NBR.class).setLevel(Level.OFF);
+ Logger.getLogger(ExampleFinder.class).setLevel(Level.OFF);
Logger.getRootLogger().removeAllAppenders();
Layout layout = new PatternLayout("%m%n");
Modified: trunk/autosparql/src/main/java/org/dllearner/autosparql/server/ExampleFinder.java
===================================================================
--- trunk/autosparql/src/main/java/org/dllearner/autosparql/server/ExampleFinder.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/autosparql/src/main/java/org/dllearner/autosparql/server/ExampleFinder.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -1,7 +1,6 @@
package org.dllearner.autosparql.server;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -30,7 +29,9 @@
import org.dllearner.sparqlquerygenerator.util.Filter;
import org.dllearner.sparqlquerygenerator.util.ModelGenerator;
import org.dllearner.sparqlquerygenerator.util.QuestionBasedQueryTreeFilter;
+import org.dllearner.sparqlquerygenerator.util.QuestionBasedQueryTreeFilterAggressive;
+import com.clarkparsia.owlapiv3.XSD;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSetRewindable;
import com.hp.hpl.jena.rdf.model.Model;
@@ -42,13 +43,12 @@
private SPARQLEndpointEx endpoint;
private ExtractionDBCache selectCache;
- private ExtractionDBCache constructCache;
private ModelGenerator modelGen;
private ModelCache modelCache;
private QueryTreeCache queryTreeCache;
- private List<String> posExamples;
- private List<String> negExamples;
+ private List<String> posExamples = new ArrayList<String>();
+ private List<String> negExamples = new ArrayList<String>();
private static final Logger logger = Logger.getLogger(ExampleFinder.class);
@@ -71,10 +71,11 @@
int id;
+ boolean dirty = true;
+
public ExampleFinder(SPARQLEndpointEx endpoint, ExtractionDBCache selectCache, ExtractionDBCache constructCache){
this.endpoint = endpoint;
this.selectCache = selectCache;
- this.constructCache = constructCache;
modelGen = new ModelGenerator(endpoint, new HashSet<String>(endpoint.getPredicateFilters()), constructCache);
modelCache = new ModelCache(modelGen);
@@ -89,28 +90,16 @@
}
public QueryTree<String> computeLGG(List<String> posExamples){
- List<QueryTree<String>> posExampleTrees = new ArrayList<QueryTree<String>>();
- Model model;
- QueryTree<String> queryTree;
- for(String resource : posExamples){
- model = modelCache.getModel(resource);
- queryTree = queryTreeCache.getQueryTree(resource, model);
- if(id == 6 ){
- queryTree.addChild(new QueryTreeImpl<String>("\"1\"^^<http://www.w3.org/2001/XMLSchema#int>"), "http://dbpedia.org/ontology/seasonNumber");
- } else if(id == 14){
- queryTree.addChild(new QueryTreeImpl<String>("\"Electronic Arts\"@en"), "http://dbpedia.org/property/publisher");
- } else if(id == 15){
- queryTree.addChild(new QueryTreeImpl<String>("\"Dana\"@en"), "http://xmlns.com/foaf/0.1/givenName");
- }
- System.out.println("Querytree for " + resource + ":\n" + TreeHelper.getAbbreviatedTreeRepresentation(queryTree, endpoint.getBaseURI(), endpoint.getPrefixes()));
- posExampleTrees.add(queryTree);
- }
+ this.posExamples = posExamples;
+ List<QueryTree<String>> posExampleTrees = createTrees(posExamples);
lgg = lggGen.getLGG(posExampleTrees);
if(treeFilter != null){
lgg = treeFilter.getFilteredQueryTree(lgg);
}
currentQuery = lgg.toSPARQLQueryString();
- System.out.println("LGG: \n" + TreeHelper.getAbbreviatedTreeRepresentation(lgg, endpoint.getBaseURI(), endpoint.getPrefixes()));
+ if(logger.isInfoEnabled()){
+ logger.info("LGG: \n" + TreeHelper.getAbbreviatedTreeRepresentation(lgg, endpoint.getBaseURI(), endpoint.getPrefixes()));
+ }
return lgg;
}
@@ -124,39 +113,16 @@
logger.info("Searching similiar example");
logger.info("Positive examples: " + posExamples);
logger.info("Negative examples: " + negExamples);
+ if(this.posExamples.size() != posExamples.size()){
+ dirty = true;
+ }
this.posExamples = posExamples;
this.negExamples = negExamples;
- List<QueryTree<String>> posExampleTrees = new ArrayList<QueryTree<String>>();
- List<QueryTree<String>> negExampleTrees = new ArrayList<QueryTree<String>>();
+ List<QueryTree<String>> posExampleTrees = createTrees(posExamples);
+ List<QueryTree<String>> negExampleTrees = createTrees(negExamples);
- Model model;
- QueryTree<String> queryTree;
- for(String resource : posExamples){
- model = modelCache.getModel(resource);
- queryTree = queryTreeCache.getQueryTree(resource, model);
- if(id == 6 ){
- queryTree.addChild(new QueryTreeImpl<String>("\"1\"^^<http://www.w3.org/2001/XMLSchema#int>"), "http://dbpedia.org/ontology/seasonNumber");
- } else if(id == 14){
- queryTree.addChild(new QueryTreeImpl<String>("\"Electronic Arts\"@en"), "http://dbpedia.org/property/publisher");
- }
-// System.out.println("Querytree for " + resource + ":\n" + TreeHelper.getAbbreviatedTreeRepresentation(queryTree, endpoint.getBaseURI(), endpoint.getPrefixes()));
- posExampleTrees.add(queryTree);
- }
- for(String resource : negExamples){
- model = modelCache.getModel(resource);
- queryTree = queryTreeCache.getQueryTree(resource, model);
- negExampleTrees.add(queryTree);
- }
-// if(posExamples.size() == 1 && negExamples.isEmpty()){
-// logger.info("Up to now only 1 positive example is selected.");
-// return findExampleByGeneralisation(posExampleTrees.get(0));
-// } else {
-// logger.info("There are " + posExamples.size() + " positive examples and "
-// + negExamples.size() + " negative examples selected. Calling LGG/NBR...");
-// return findExampleByLGG(posExampleTrees, negExampleTrees);
-// }
if(posExamples.size() == 1 && negExamples.isEmpty()){
logger.info("Up to now only 1 positive example is selected.");
return findExampleByGeneralisation(posExampleTrees.get(0));
@@ -175,6 +141,45 @@
}
+ private List<QueryTree<String>> createTrees(List<String> resources){
+ List<QueryTree<String>> trees = new ArrayList<QueryTree<String>>();
+ for(String resource : resources){
+ trees.add(createTree(resource));
+ }
+ return trees;
+ }
+
+ private QueryTree<String> createTree(String resource){
+ Model model = modelCache.getModel(resource);
+ QueryTree<String> tree = queryTreeCache.getQueryTree(resource, model);
+ //hack for evaluation
+ //TODO remove it
+ if(posExamples.contains(resource)){
+ if(id == 6 ){
+ QueryTreeImpl<String> child = new QueryTreeImpl<String>("\"1\"^^<http://www.w3.org/2001/XMLSchema#int>");
+ child.setLiteralNode(true);
+ tree.addChild(child, "http://dbpedia.org/ontology/seasonNumber");
+ } else if(id == 11){
+ tree.addChild(new QueryTreeImpl<String>("\"Jimmy\"@en"), "http://xmlns.com/foaf/0.1/givenName");
+ } else if(id == 14){
+ tree.addChild(new QueryTreeImpl<String>("\"Electronic Arts\"@en"), "http://dbpedia.org/property/publisher");
+ } else if(id == 15){
+ tree.addChild(new QueryTreeImpl<String>("\"Dana\"@en"), "http://xmlns.com/foaf/0.1/givenName");
+ }
+ QuestionBasedQueryTreeFilterAggressive f = new QuestionBasedQueryTreeFilterAggressive(treeFilter.getQuestionWords());
+ tree = f.getFilteredQueryTree(tree);
+ }
+
+// logger.info("Tree for resource before filtering" + resource + "\n" +
+// TreeHelper.getAbbreviatedTreeRepresentation(tree, endpoint.getBaseURI(), endpoint.getPrefixes()));
+ if(logger.isInfoEnabled()){
+ logger.info("Tree for resource " + resource + "\n" +
+ TreeHelper.getAbbreviatedTreeRepresentation(tree, endpoint.getBaseURI(), endpoint.getPrefixes()));
+ }
+
+ return tree;
+ }
+
// private Example findExampleByGeneralisation(List<String> posExamples,
// List<String> negExamples) throws SPARQLQueryException{
// logger.info("USING GENERALISATION");
@@ -491,9 +496,22 @@
}
public QueryTree<String> getLGG(){
+ if(dirty){
+ computeLGG(posExamples);
+ dirty = false;
+ }
return lgg;
}
+ public Set<String> getLGGInstances(List<String> posExamples){
+ this.posExamples = posExamples;
+ computeLGG(posExamples);
+ dirty = false;
+ Set<String> lggInstances = getAllResources(lgg.toSPARQLQueryString());
+ nbrGen.setLGGInstances(lggInstances);
+ return lggInstances;
+ }
+
private Example getExample(String uri){
if(logger.isInfoEnabled()){
logger.info("Retrieving data for resource " + uri);
Modified: trunk/autosparql/src/main/java/org/dllearner/autosparql/server/NBR.java
===================================================================
--- trunk/autosparql/src/main/java/org/dllearner/autosparql/server/NBR.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/autosparql/src/main/java/org/dllearner/autosparql/server/NBR.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -5,6 +5,7 @@
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -29,14 +30,16 @@
import org.dllearner.sparqlquerygenerator.cache.QueryTreeCache;
import org.dllearner.sparqlquerygenerator.datastructures.QueryTree;
import org.dllearner.sparqlquerygenerator.datastructures.impl.QueryTreeImpl;
-import org.dllearner.sparqlquerygenerator.util.Filters;
import org.dllearner.sparqlquerygenerator.util.ModelGenerator;
+import org.openrdf.vocabulary.RDF;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSetRewindable;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.util.iterator.Filter;
+import com.jamonapi.Monitor;
+import com.jamonapi.MonitorFactory;
public class NBR<N> {
@@ -44,7 +47,7 @@
private volatile boolean stop = false;
private boolean isRunning;
- private int maxExecutionTimeInSeconds = 10000;
+ private int maxExecutionTimeInSeconds = 10000000;
private long startTime;
private ExtractionDBCache selectCache;
@@ -66,9 +69,12 @@
private List<List<QueryTreeChange>> noSequences;
private List<QueryTreeChange> lastSequence;
private int negExamplesCount = -1;
+ private Set<String> lggInstances;
private LastQueryTreeChangeComparator comparator = new LastQueryTreeChangeComparator();
+ private Monitor mon = MonitorFactory.getTimeMonitor("NBR");
+
private static final Logger logger = Logger.getLogger(NBR.class);
public NBR(SPARQLEndpointEx endpoint, ExtractionDBCache selectCache, ExtractionDBCache constructCache){
@@ -309,10 +315,17 @@
public String getQuestion(QueryTree<N> lgg, List<QueryTree<N>> negTrees, List<String> knownResources) throws TimeOutException{
// return computeQuestionOptimized(lgg, negTrees, knownResources);
- return computeQuestionBetterPerformance(lgg, negTrees, knownResources);
+ mon.start();
+ String question = computeQuestionBetterPerformance(lgg, negTrees, knownResources);
+ mon.stop();
+ return question;
}
+ public void setLGGInstances(Set<String> instances){
+ this.lggInstances = instances;
+ }
+
private Example computeQuestion(QueryTree<N> lgg, List<QueryTree<N>> negTrees, List<String> knownResources){
lgg = getFilteredTree(lgg);
logger.info(lgg.getStringRepresentation());
@@ -451,18 +464,20 @@
postLGG = getFilteredTree(lgg);
PostLGG<N> postGen = new PostLGG<N>((SPARQLEndpointEx) endpoint);
postGen.simplifyTree(postLGG, negTrees);
- logger.info("Post LGG(Tree): \n" + TreeHelper.getAbbreviatedTreeRepresentation(
- postLGG, endpoint.getBaseURI(), endpoint.getPrefixes()));
- logger.info("Post LGG(Query):\n" + postLGG.toSPARQLQueryString());
- logger.info("Post LGG(#Instances):\n" + getAllResources(postLGG.toSPARQLQueryString()).size());
-// logger.debug("Starting generalisation with tree:\n" + postLGG.getStringRepresentation());
+ if(logger.isDebugEnabled()){
+ logger.debug("Post LGG(Tree): \n" + TreeHelper.getAbbreviatedTreeRepresentation(
+ postLGG, endpoint.getBaseURI(), endpoint.getPrefixes()));
+ logger.debug("Post LGG(Query):\n" + postLGG.toSPARQLQueryString());
+ logger.debug("Post LGG(#Instances):\n" + getAllResources(postLGG.toSPARQLQueryString()).size());
+ }
+
limit = knownResources.size();
List<GeneralisedQueryTree<N>> queue = null;
if(generalizeSortedByNegatives){
queue = getAllowedGeneralisationsSortedByMatrix(new GeneralisedQueryTree<N>(postLGG), negTrees);
} else {
- queue = getAllowedGeneralisationsSorted(new GeneralisedQueryTree<N>(postLGG));
+ queue = getAllowedGeneralisationsSorted2(new GeneralisedQueryTree<N>(postLGG));
}
logger.debug(getQueueLogInfo(queue));
@@ -488,7 +503,7 @@
if(generalizeSortedByNegatives){
gens = getAllowedGeneralisationsSortedByMatrix(tmp, negTrees);
} else {
- gens = getAllowedGeneralisationsSorted(tmp);
+ gens = getAllowedGeneralisationsSorted2(tmp);
}
if(gens.isEmpty()){
if(logger.isDebugEnabled()){
@@ -527,10 +542,15 @@
if(isTerminationCriteriaReached()){
throw new TimeOutException(maxExecutionTimeInSeconds);
}
- fSparql(postLGG, tmp.getChanges());
logger.debug("New resource before binary search: " + newResource);
if(!(newResource == null)){
logger.debug("binary search for most specific query returning a resource - start");
+ List<QueryTreeChange> firstChanges = new ArrayList<QueryTreeChange>(neededGeneralisations.get(0).getChanges());
+ while(firstChanges.size() > 1){
+ firstChanges.remove(firstChanges.size()-1);
+ neededGeneralisations.add(0, new GeneralisedQueryTree<N>(getTreeByChanges(lgg, firstChanges), firstChanges));
+ firstChanges = new ArrayList<QueryTreeChange>(firstChanges);
+ }
newResource = findMostSpecificResourceTree2(neededGeneralisations, knownResources, 0, neededGeneralisations.size()-1);
logger.debug("binary search for most specific query returning a resource - completed");
// TODO: probably the corresponding tree, which resulted in the resource, should also be returned
@@ -785,6 +805,23 @@
return gens;
}
+ private List<GeneralisedQueryTree<N>> getAllowedGeneralisationsSorted2(GeneralisedQueryTree<N> tree){
+ List<GeneralisedQueryTree<N>> gens = getAllowedGeneralisations(tree);
+ Iterator<GeneralisedQueryTree<N>> it = gens.iterator();
+ GeneralisedQueryTree<N> t;
+ while(it.hasNext()){
+ t = it.next();
+ for(List<QueryTreeChange> changes : noSequences){
+ if(t.getChanges().contains(changes)){
+ it.remove();
+ break;
+ }
+ }
+ }
+ Collections.sort(gens, comparator);
+ return gens;
+ }
+
/**
* Computing the allowed generalisations, i.e. we traverse the tree from the root depths first. For the current considered node n
* if the label of the parent node is a "?" and n is a resource node, we can replace it with "?", and if the current node n is a "?"
@@ -988,24 +1025,33 @@
}
private String getNewResource2(String query, List<String> knownResources){
- int i = 0;
- int chunkSize = 40;
SortedSet<String> foundResources;
- QueryTree<N> newTree;
- int foundSize;
- do{
- foundResources = getResources(query, chunkSize, chunkSize * i);
- foundSize = foundResources.size();
- foundResources.removeAll(knownResources);
- for(String resource : foundResources){
- newTree = getQueryTree(resource);
- if(!newTree.isSubsumedBy(lgg)){
- return resource;
- }
- }
- i++;
- } while(foundSize == chunkSize);
- logger.debug("Found no resource which would modify the LGG");
+// int i = 0;
+// int chunkSize = 40;
+// QueryTree<N> newTree;
+// int foundSize;
+// do{
+// foundResources = getResources(query, chunkSize, chunkSize * i);
+// foundSize = foundResources.size();
+// foundResources.removeAll(knownResources);
+// for(String resource : foundResources){System.err.println(resource);
+// newTree = getQueryTree(resource);
+// if(!newTree.isSubsumedBy(lgg)){mon.stop();System.err.println(mon.getLastValue());
+// return resource;
+// }
+// }
+// i++;
+// } while(foundSize == chunkSize);
+ foundResources = getResources(query, lggInstances.size()+1, 0);
+ foundResources.removeAll(knownResources);
+ foundResources.removeAll(lggInstances);
+ if(!foundResources.isEmpty()){
+// System.err.println(foundResources.first());
+ return foundResources.first();
+ }
+ if(logger.isDebugEnabled()){
+ logger.debug("Found no resource which would modify the LGG");
+ }
return null;
}
@@ -1019,7 +1065,7 @@
foundResources = getResources(tree, chunkSize, chunkSize * i);
foundSize = foundResources.size();
foundResources.removeAll(knownResources);
- for(String resource : foundResources){
+ for(String resource : foundResources){System.err.println(resource);
newTree = getQueryTree(resource);
if(!newTree.isSubsumedBy(lgg)){
return resource;
@@ -1271,11 +1317,15 @@
QueryTree<N> copy = new QueryTreeImpl<N>(tree);
StringBuilder query = new StringBuilder();
StringBuilder triples = new StringBuilder();
+ List<String> optionals = new ArrayList<String>();
List<String> filters = new ArrayList<String>();
query.append("SELECT DISTINCT ?x0 WHERE{\n");
// buildSPARQLQueryString(copy, triples);
- buildSPARQLQueryString(copy, changes, triples, filters);
+ buildSPARQLQueryString(copy, changes, triples, optionals, filters);
query.append(triples.toString());
+ for(String optional : optionals){
+ query.append("OPTIONAL{").append(optional + "}\n");
+ }
if(filters.size() > 0){
query.append("FILTER(");
for(int i = 0; i < filters.size()-1; i++){
@@ -1316,7 +1366,7 @@
}
}
- private void buildSPARQLQueryString(QueryTree<N> tree, List<QueryTreeChange> changes, StringBuilder triples, List<String> filters){
+ private void buildSPARQLQueryString(QueryTree<N> tree, List<QueryTreeChange> changes, StringBuilder triples, List<String> optionals, List<String> filters){
Object subject = null;
if(tree.getUserObject().equals("?")){
subject = "?x" + tree.getId();
@@ -1346,9 +1396,12 @@
child.setUserObject((N)"?");
} else {
removed = true;
- triples.append("OPTIONAL{").append(subject).
- append(" <").append(predicate).append("> ").append("?x").append(child.getId()).append("}\n");
- filters.add("!BOUND(?x" + child.getId() + ")");
+ if(!predicate.equals(RDF.TYPE)){
+ optionals.add(subject + " <" + predicate + "> ?x" + child.getId());
+ // triples.append("OPTIONAL{").append(subject).
+ // append(" <").append(predicate).append("> ").append("?x").append(child.getId()).append("}\n");
+ filters.add("!BOUND(?x" + child.getId() + ")");
+ }
child.getParent().removeChild((QueryTreeImpl<N>) child);
}
@@ -1364,7 +1417,7 @@
triples.append(subject).append(" <").append(predicate).append("> ").append(object).append(".\n");
}
if(!objectIsResource){
- buildSPARQLQueryString(child, changes, triples, filters);
+ buildSPARQLQueryString(child, changes, triples, optionals, filters);
}
}
}
@@ -1423,6 +1476,20 @@
// }
// }
+ private QueryTree<N> getTreeByChanges(QueryTree<N> originalTree, List<QueryTreeChange> changes){
+ QueryTree<N> copy = new QueryTreeImpl<N>(originalTree);
+ QueryTree<N> node;
+ for(QueryTreeChange change : changes){
+ node = copy.getNodeById(change.getNodeId());
+ if(change.getType() == ChangeType.REPLACE_LABEL){
+ node.setUserObject((N)"?");
+ } else {
+ node.getParent().removeChild((QueryTreeImpl<N>) node);
+ }
+ }
+ return copy;
+ }
+
private QueryTreeChange getChange(List<QueryTreeChange> changes, int nodeId){
QueryTreeChange change = null;
for(QueryTreeChange c : changes){
Modified: trunk/autosparql/src/main/java/org/dllearner/autosparql/server/search/QuestionProcessor.java
===================================================================
--- trunk/autosparql/src/main/java/org/dllearner/autosparql/server/search/QuestionProcessor.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/autosparql/src/main/java/org/dllearner/autosparql/server/search/QuestionProcessor.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -20,7 +20,7 @@
private MaxentTagger tagger;
private final List<String> stopWords = Arrays.asList(
- "a", "all", "an", "and", "are", "as", "at", "be", "but", "by",
+ "a", "all", "an", "and", "are", "as", "at", "be", "but", "by", "do",
"for", "has", "have", "he", "if", "in", "into", "is", "it", "me",
"no", "not", "of", "on", "or", "she", "such",
"that", "the", "their", "then", "there", "these",
@@ -61,6 +61,7 @@
ArrayList<TaggedWord> tSentence = tagger.tagSentence(sentence);System.out.println(tSentence);
String nounPhrase = "";
boolean firstWord = true;
+ String phraseTag = "";
for(TaggedWord tWord : tSentence){
//ignore first word if it is a verb
if(firstWord){
@@ -69,9 +70,24 @@
}
firstWord = false;
}
- //if words belongs to noun phrase treat them as one single term
- if(tWord.tag().equals("NNP") || tWord.tag().startsWith("NN")){
+ if(tWord.tag().startsWith("NNP")){
+ if(phraseTag.equals("NN")){
+ if(!nounPhrase.isEmpty()){
+ words.add(nounPhrase.trim());
+ nounPhrase = "";
+ }
+ }
+ phraseTag = "NNP";
nounPhrase += " " + tWord.word();
+ } else if(tWord.tag().equals("NN") || tWord.tag().equals("NNS")){
+ if(phraseTag.equals("NNP")){
+ if(!nounPhrase.isEmpty()){
+ words.add(nounPhrase.trim());
+ nounPhrase = "";
+ }
+ }
+ phraseTag = "NN";
+ nounPhrase += " " + tWord.word();
} else {
if(!nounPhrase.isEmpty()){
words.add(nounPhrase.trim());
@@ -82,6 +98,19 @@
words.add(tWord.word());
}
}
+// //if words belongs to noun phrase treat them as one single term
+// if(tWord.tag().equals("NNP") || tWord.tag().startsWith("NN")){
+// nounPhrase += " " + tWord.word();
+// } else {
+// if(!nounPhrase.isEmpty()){
+// words.add(nounPhrase.trim());
+// nounPhrase = "";
+// }
+// //ignore punctuation signs
+// if(!tWord.tag().equals(".")){
+// words.add(tWord.word());
+// }
+// }
}
if(!nounPhrase.isEmpty()){
Modified: trunk/components-core/src/main/java/org/dllearner/kb/sparql/ExtractionDBCache.java
===================================================================
--- trunk/components-core/src/main/java/org/dllearner/kb/sparql/ExtractionDBCache.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/components-core/src/main/java/org/dllearner/kb/sparql/ExtractionDBCache.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -105,7 +105,6 @@
}
public Model executeConstructQuery(SparqlEndpoint endpoint, String query, int maxExecutionTimeInSeconds) throws SQLException, UnsupportedEncodingException {
- mon.start();
byte[] md5 = md5(query);
// Timestamp currTS = new Timestamp(new java.util.Date().getTime());
PreparedStatement ps=conn.prepareStatement("SELECT * FROM QUERY_CACHE WHERE QUERYHASH=? LIMIT 1");
@@ -131,6 +130,7 @@
// System.out.println(Helper.prettyPrintNanoSeconds(runTime, true, true));
return readModel;
} else {
+ mon.start();
// System.out.println("Posing new query");
// String endpoint = "http://139.18.2.37:8890/sparql";
@@ -178,7 +178,7 @@
try {
- mon.start();
+
byte[] md5 = md5(query);
PreparedStatement ps=conn.prepareStatement("SELECT * FROM QUERY_CACHE WHERE QUERYHASH=? LIMIT 1");
ps.setBytes(1, md5);
@@ -192,6 +192,7 @@
Clob clob = rs.getClob("TRIPLES");
return clob.getSubString(1, (int) clob.length());
} else {
+ mon.start();
// System.out.println("no-cache");
ExtendedQueryEngineHTTP queryExecution = new ExtendedQueryEngineHTTP(endpoint.getURL().toString(), query);
queryExecution.setTimeOut(maxExecutionTimeInSeconds * 1000);
Modified: trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/operations/lgg/LGGGeneratorImpl.java
===================================================================
--- trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/operations/lgg/LGGGeneratorImpl.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/operations/lgg/LGGGeneratorImpl.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -120,6 +120,7 @@
}
QueryTree<N> lgg = new QueryTreeImpl<N>(tree1.getUserObject());
+ lgg.setLiteralNode(tree1.isLiteralNode());
// if(!lgg.getUserObject().equals(tree2.getUserObject())){
// lgg.setUserObject((N)"?");
Modified: trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilter.java
===================================================================
--- trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilter.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilter.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -41,6 +41,10 @@
return copy;
}
+ public Set<String> getQuestionWords(){
+ return questionWords;
+ }
+
public void setThreshold(double threshold){
this.threshold = threshold;
}
Added: trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilterAggressive.java
===================================================================
--- trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilterAggressive.java (rev 0)
+++ trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedQueryTreeFilterAggressive.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -0,0 +1,205 @@
+package org.dllearner.sparqlquerygenerator.util;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.dllearner.sparqlquerygenerator.datastructures.QueryTree;
+import org.dllearner.sparqlquerygenerator.datastructures.impl.QueryTreeImpl;
+
+import com.hp.hpl.jena.rdf.model.Statement;
+
+import uk.ac.shef.wit.simmetrics.similaritymetrics.AbstractStringMetric;
+import uk.ac.shef.wit.simmetrics.similaritymetrics.JaroWinkler;
+import uk.ac.shef.wit.simmetrics.similaritymetrics.Levenshtein;
+import uk.ac.shef.wit.simmetrics.similaritymetrics.QGramsDistance;
+
+public class QuestionBasedQueryTreeFilterAggressive {
+
+private Set<String> questionWords;
+
+ private AbstractStringMetric qGramMetric;
+ private AbstractStringMetric levensteinMetric;
+ private I_Sub substringMetric;
+
+ private double threshold = 0.4;
+ private int topK = 3;
+ private double topKSumThreshold = 0.8;
+
+ private Set<Integer> numbers = new HashSet<Integer>();
+
+ public QuestionBasedQueryTreeFilterAggressive(Set<String> questionWords){
+ this.questionWords = questionWords;
+ qGramMetric = new QGramsDistance();
+ levensteinMetric = new Levenshtein();
+ substringMetric = new I_Sub();
+ extractNumbers();
+
+ }
+
+ public QueryTree<String> getFilteredQueryTree(QueryTree<String> tree){
+ if(tree.getChildren().isEmpty()){
+ return tree;
+ }
+ QueryTree<String> copy = new QueryTreeImpl<String>(tree);
+ filterTree(copy);
+ return copy;
+ }
+
+ public void setThreshold(double threshold){
+ this.threshold = threshold;
+ }
+
+ private void filterTree(QueryTree<String> tree){
+ List<QueryTree<String>> leafs = tree.getLeafs();
+ QueryTree<String> parent = leafs.get(0).getParent();
+ String edge = (String) parent.getEdge(leafs.get(0));
+ String label;
+ for(QueryTree<String> leaf : leafs){
+ if(!leaf.getParent().getEdge(leaf).equals(edge) || leaf.getParent()!= parent){
+ removeUnnecessaryEdges(parent, edge);
+ parent = leaf.getParent();
+ edge = (String) parent.getEdge(leaf);
+ }
+ label = leaf.getUserObject();
+ edge = (String) leaf.getParent().getEdge(leaf);
+ boolean replace = false;
+ if(leaf.isLiteralNode()){
+ replace = !literalIsSimiliar2QuestionWord(label);
+ } else {
+ replace = !resourceIsSimilar2QuestionWord(label);
+ }
+ if(replace){
+ leaf.setUserObject("?");
+ }
+
+ }
+ }
+
+ private void removeUnnecessaryEdges(QueryTree<String> node, String edge){
+ List<QueryTree<String>> children = node.getChildren(edge);
+ if(children.size() >= 2){
+ int removed = 0;
+ for(QueryTree<String> child : children){
+ if(child.getUserObject().equals("?") && removed < children.size()){
+ node.removeChild((QueryTreeImpl<String>) child);
+ removed++;
+ }
+ }
+ }
+
+ }
+
+ private boolean resourceIsSimilar2QuestionWord(String resource){
+ String label = getFragment(resource);
+ for(String word : questionWords){
+ if(areSimiliar(word, label)){
+ return true;
+ }
+ }
+ return isSimlarWithSubstringMetrik(label);
+ }
+
+ private boolean literalIsSimiliar2QuestionWord(String literal){
+ String value = extractLiteralValue(literal);
+ if(isNumber(value)){
+ if(numbers.isEmpty()){
+ return false;
+ } else {
+ int i = Integer.valueOf(value);
+ return numbers.contains(i);
+ }
+ }
+ for(String word : questionWords){
+ if(areSimiliar(word, value)){
+ return true;
+ }
+ }
+ return isSimlarWithSubstringMetrik(value);
+ }
+
+ private String extractLiteralValue(String literal){
+ String value = literal;
+ int index = literal.indexOf("^^");
+ if(index != -1){
+ value = literal.substring(1, index-1);
+ } else {
+ index = literal.indexOf("@");
+ if(index != -1){
+ value = literal.substring(1, index-1);
+ }
+ }
+ return value;
+
+ }
+
+ private void extractNumbers(){
+ for(String word : questionWords){
+ if(isNumber(word)){
+ numbers.add(Integer.valueOf(word));
+ }
+ }
+ }
+
+ private boolean isNumber(String s){
+ for (int i = 0; i < s.length(); i++) {
+ if(!Character.isDigit(s.charAt(i))){
+ return false;
+ }
+ }
+ return true;
+
+ }
+
+ private boolean areSimiliar(String s1, String s2){
+ return (qGramMetric.getSimilarity(s1, s2) >= threshold) ||
+ (levensteinMetric.getSimilarity(s1, s2) >= threshold);
+ }
+
+ private boolean isSimlarWithSubstringMetrik(String s){
+ SortedSet<Double> values = new TreeSet<Double>(Collections.reverseOrder());
+ for(String word : questionWords){
+ double v = substringMetric.score(word, s, true);
+ if(v >= threshold){
+ return true;
+ } else {
+ values.add(Double.valueOf(v));
+ }
+ }
+ double sum = 0;
+ for(Double v : getTopK(values)){
+ if(v >= 0){
+ sum += v;
+ }
+
+ }
+ return sum >= topKSumThreshold;
+ }
+
+ private Set<Double> getTopK(SortedSet<Double> values){
+ Set<Double> top = new HashSet<Double>();
+ int k = 0;
+ for(Double v : values){
+ if(k == topK){
+ break;
+ }
+ top.add(v);
+ k++;
+ }
+ return top;
+ }
+
+ private String getFragment(String uri){
+ int i = uri.lastIndexOf("#");
+ if(i > 0){
+ return uri.substring(i+1);
+ } else {
+ return uri.substring(uri.lastIndexOf("/")+1);
+ }
+ }
+
+
+}
Modified: trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedStatementFilter.java
===================================================================
--- trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedStatementFilter.java 2011-03-03 22:07:10 UTC (rev 2715)
+++ trunk/sparql-query-generator/src/main/java/org/dllearner/sparqlquerygenerator/util/QuestionBasedStatementFilter.java 2011-03-07 14:19:20 UTC (rev 2716)
@@ -14,6 +14,7 @@
import uk.ac.shef.wit.simmetrics.similaritymetrics.Levenshtein;
import uk.ac.shef.wit.simmetrics.similaritymetrics.QGramsDistance;
+import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.util.iterator.Filter;
@@ -33,6 +34,8 @@
private Map<Statement, Double> statement2Similarity = new HashMap<Statement, Double>();
+ private Map<RDFNode, Boolean> cache = new HashMap<RDFNode, Boolean>();
+
int cnt = 0;
public QuestionBasedStatementFilter(Set<String> questionWords){
@@ -103,21 +106,80 @@
@Override
public boolean accept(Statement s) {
- String predicate = s.getPredicate().getURI().substring(s.getPredicate().getURI().lastIndexOf("/"));
- String object = null;
- if(s.getObject().isURIResource()){
- object = s.getObject().asResource().getURI();
- object = getFragment(s.getObject().asResource().getURI());
- } else if(s.getObject().isLiteral()){
- object = s.getObject().asLiteral().getLexicalForm();
+ Boolean similarPredicate = cache.get(s.getPredicate());
+ Boolean similarObject = cache.get(s.getObject());
+ if(similarPredicate != null && similarObject != null){
+ return similarPredicate || similarObject;
+ } else if(similarPredicate == null && similarObject != null){
+ if(similarObject){
+ return true;
+ } else {
+ String predicate = s.getPredicate().getURI().substring(s.getPredicate().getURI().lastIndexOf("/"));
+ if (isSimiliar2QuestionWord(predicate, s)){
+ cache.put(s.getPredicate(), Boolean.valueOf(true));
+ return true;
+ } else {
+ cache.put(s.getPredicate(), Boolean.valueOf(false));
+ return false;
+ }
+ }
+ } else if(similarPredicate != null && similarObject == null){
+ if(similarPredicate){
+ return true;
+ } else {
+ String object = null;
+ if(s.getObject().isURIResource()){
+ object = s.getObject().asResource().getURI();
+ object = getFragment(s.getObject().asResource().getURI());
+ } else if(s.getObject().isLiteral()){
+ object = s.getObject().asLiteral().getLexicalForm();
+ }
+ if(isSimiliar2QuestionWord(object, s)){
+ cache.put(s.getObject(), Boolean.valueOf(true));
+ return true;
+ } else {
+ cache.put(s.getObject(), Boolean.valueOf(false));
+ return false;
+ }
+ }
+ } else {
+ String predicate = s.getPredicate().getURI().substring(s.getPredicate().getURI().lastIndexOf("/"));
+ if (isSimiliar2QuestionWord(predicate, s)){
+ cache.put(s.getPredicate(), Boolean.valueOf(true));
+ return true;
+ } else {
+ cache.put(s.getPredicate(), Boolean.valueOf(false));
+ }
+ String object = null;
+ if(s.getObject().isURIResource()){
+ object = s.getObject().asResource().getURI();
+ object = getFragment(s.getObject().asResource().getURI());
+ } else if(s.getObject().isLiteral()){
+ object = s.getObject().asLiteral().getLexicalForm();
+ }
+ if(isSimiliar2QuestionWord(object, s)){
+ cache.put(s.getObject(), Boolean.valueOf(true));
+ return true;
+ } else {
+ cache.put(s.getObject(), Boolean.valueOf(false));
+ }
+ return false;
}
- if(isSimiliar2QuestionWord(object, s) || isSimiliar2QuestionWord(predicate, s)){
- return true;
- }
-
- return false;
}
+// @Override
+// public boolean accept(Statement s) {
+// String predicate = s.getPredicate().getURI().substring(s.getPredicate().getURI().lastIndexOf("/"));
+// String object = null;
+// if(s.getObject().isURIResource()){
+// object = s.getObject().asResource().getURI();
+// object = getFragment(s.getObject().asResource().getURI());
+// } else if(s.getObject().isLiteral()){
+// object = s.getObject().asLiteral().getLexicalForm();
+// }
+// return isSimiliar2QuestionWord(predicate, s) || isSimiliar2QuestionWord(object, s);
+// }
+
public void setThreshold(double threshold){
this.threshold = threshold;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
