From: <lor...@us...> - 2014-02-03 14:04:44
|
Revision: 4217 http://sourceforge.net/p/dl-learner/code/4217 Author: lorenz_b Date: 2014-02-03 14:04:40 +0000 (Mon, 03 Feb 2014) Log Message: ----------- Added method to get transitive closure of sublcasses via SPARQL 1.1 Modified Paths: -------------- trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithm.java trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithmDisjunctive.java trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/StableHeuristic.java trunk/components-core/src/main/java/org/dllearner/core/AbstractReasonerComponent.java trunk/components-core/src/main/java/org/dllearner/reasoning/FastInstanceChecker.java trunk/components-core/src/main/java/org/dllearner/reasoning/OWLAPIReasoner.java trunk/components-core/src/main/java/org/dllearner/reasoning/SPARQLReasoner.java trunk/components-core/src/main/java/org/dllearner/refinementoperators/ELDown3.java trunk/components-core/src/test/java/org/dllearner/algorithms/isle/DBpediaExperiment.java trunk/components-core/src/test/java/org/dllearner/algorithms/isle/KnowledgebaseSampleGenerator.java Modified: trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithm.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithm.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithm.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -229,6 +229,7 @@ } else { node.setCoveredNegatives(negCovers); } + node.setScore(accuracy); // link to parent (unless start node) if(parentNode == null) { startNode = node; @@ -316,6 +317,17 @@ if(occursOnFirstLevel(description, classToDescribe)) { return false; } + + //non of the equivalent classes must occur on the first level + TreeSet<Description> toTest = new TreeSet<Description>(descriptionComparator); + toTest.add(classToDescribe); + while(!toTest.isEmpty()) { + Description d = toTest.pollFirst(); + if(occursOnFirstLevel(description, d)) { + return false; + } + toTest.addAll(reasoner.getEquivalentClasses(d)); + } } else { // none of the superclasses of the class to learn must appear on the // outermost property level Modified: trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithmDisjunctive.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithmDisjunctive.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/ELLearningAlgorithmDisjunctive.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -38,6 +38,7 @@ import org.dllearner.core.config.ConfigOption; import org.dllearner.core.owl.Description; import org.dllearner.core.owl.Individual; +import org.dllearner.core.owl.NamedClass; import org.dllearner.core.owl.Thing; import org.dllearner.core.owl.Union; import org.dllearner.learningproblems.PosNegLP; @@ -94,12 +95,13 @@ private boolean isRunning = false; private boolean stop = false; - private Description startClass; private SearchTreeNode startNode; private ELHeuristic heuristic; private TreeSet<SearchTreeNode> candidates; // all trees (for fast redundancy check) private TreeSet<ELDescriptionTree> trees; + private NamedClass classToDescribe; + private double noise; @ConfigOption(name = "treeSearchTimeSeconds", defaultValue = "1.0", description="Specifies how long the algorithm should search for a partial solution (a tree).") private double treeSearchTimeSeconds = 1.0; @@ -107,6 +109,16 @@ @ConfigOption(name = "tryFullCoverage", defaultValue = "false", description="If yes, then the algorithm tries to cover all positive examples. Note that while this improves accuracy on the testing set, it may lead to overfitting.") private boolean tryFullCoverage = false; + @ConfigOption(name = "stopOnFirstDefinition", defaultValue="false", description="algorithm will terminate immediately when a correct definition is found") + private boolean stopOnFirstDefinition = false; + + @ConfigOption(name = "noisePercentage", defaultValue="0.0", description="the (approximated) percentage of noise within the examples") + private double noisePercentage = 0.0; + + // the class with which we start the refinement process + @ConfigOption(name = "startClass", defaultValue="owl:Thing", description="You can specify a start class for the algorithm. To do this, you have to use Manchester OWL syntax without using prefixes.") + private Description startClass; + // private double noise = 0; private List<ELDescriptionTree> currentSolution = new LinkedList<ELDescriptionTree>(); private EvaluatedDescription bestEvaluatedDescription; @@ -170,6 +182,8 @@ prefixes = reasoner.getPrefixes(); minimizer = new DescriptionMinimizer(reasoner); + + noise = noisePercentage/100d; } @Override @@ -483,5 +497,25 @@ public void setTryFullCoverage(boolean tryFullCoverage) { this.tryFullCoverage = tryFullCoverage; } - + + /** + * @return the noisePercentage + */ + public double getNoisePercentage() { + return noisePercentage; + } + + /** + * @param noisePercentage the noisePercentage to set + */ + public void setNoisePercentage(double noisePercentage) { + this.noisePercentage = noisePercentage; + } + + /** + * @param classToDescribe the classToDescribe to set + */ + public void setClassToDescribe(NamedClass classToDescribe) { + this.classToDescribe = classToDescribe; + } } Modified: trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/StableHeuristic.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/StableHeuristic.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/algorithms/elcopy/StableHeuristic.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -36,8 +36,8 @@ @Override public int compare(SearchTreeNode o1, SearchTreeNode o2) { - double diff = o2.getCoveredNegatives() - o1.getCoveredNegatives(); - + double diff = o2.getScore() - o1.getScore(); +// diff = -diff; if(diff>0) { return 1; } else if(diff<0) { Modified: trunk/components-core/src/main/java/org/dllearner/core/AbstractReasonerComponent.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/core/AbstractReasonerComponent.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/core/AbstractReasonerComponent.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -54,6 +54,8 @@ import org.dllearner.utilities.owl.OWLVocabulary; import org.dllearner.utilities.owl.RoleComparator; +import com.google.common.collect.Sets; + /** * Abstract component representing a reasoner. Only a few reasoning operations * are guaranteed to be implemented by the underlying reasoner, while a @@ -926,8 +928,12 @@ protected SortedSet<Description> getSubClassesImpl(Description concept) throws ReasoningMethodUnsupportedException { throw new ReasoningMethodUnsupportedException(); - } + } + public final SortedSet<Description> getEquivalentClasses(Description concept) { + return new TreeSet<>(Sets.intersection(getClassHierarchy().getSubClasses(concept), getClassHierarchy().getSuperClasses(concept))); + } + @Override public final SortedSet<ObjectProperty> getSuperProperties(ObjectProperty role) { return getObjectPropertyHierarchy().getMoreGeneralRoles(role); Modified: trunk/components-core/src/main/java/org/dllearner/reasoning/FastInstanceChecker.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/reasoning/FastInstanceChecker.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/reasoning/FastInstanceChecker.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -912,7 +912,7 @@ @Override protected SortedSet<Description> getSubClassesImpl(Description concept) throws ReasoningMethodUnsupportedException { return rc.getSubClassesImpl(concept); - } + } @Override protected SortedSet<ObjectProperty> getSuperPropertiesImpl(ObjectProperty role) throws ReasoningMethodUnsupportedException { Modified: trunk/components-core/src/main/java/org/dllearner/reasoning/OWLAPIReasoner.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/reasoning/OWLAPIReasoner.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/reasoning/OWLAPIReasoner.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -565,6 +565,15 @@ return getFirstClasses(classes); } + + protected SortedSet<Description> getEquivalentClassesImpl(Description concept) { + SortedSet<Description> equivalentclasses = new TreeSet<>(conceptComparator); + OWLClass cls = OWLAPIDescriptionConvertVisitor.getOWLClassExpression(concept).asOWLClass(); + for (OWLClass eqCls : reasoner.getEquivalentClasses(cls).getEntitiesMinus(cls)) { + equivalentclasses.add(new NamedClass(eqCls.toStringID())); + } + return equivalentclasses; + } @Override protected TreeSet<ObjectProperty> getSuperPropertiesImpl(ObjectProperty role) { Modified: trunk/components-core/src/main/java/org/dllearner/reasoning/SPARQLReasoner.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/reasoning/SPARQLReasoner.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/reasoning/SPARQLReasoner.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -647,6 +647,23 @@ } return types; } + + public Set<NamedClass> getMostSpecificTypes(Individual individual) { + Set<NamedClass> types = new HashSet<NamedClass>(); + String query = String.format( + "SELECT ?type WHERE {<%s> a ?type . " + + "FILTER NOT EXISTS{<%s> a ?moreSpecificType ." + + "?moreSpecificType <http://www.w3.org/2000/01/rdf-schema#subClassOf>+ ?type.}}", individual.getName(), individual.getName()); + ResultSet rs = executeSelectQuery(query); + QuerySolution qs; + while(rs.hasNext()){ + qs = rs.next(); + types.add(new NamedClass(qs.getResource("type").getURI())); + } + return types; + } + + public Set<NamedClass> getTypes() { return getTypes((String)null); @@ -1569,23 +1586,16 @@ return getSubClasses(description, false); } - public SortedSet<Description> getSubClasses(Description description, boolean useVirtuoso) { - if(!(description instanceof NamedClass || description instanceof Thing)){ - throw new IllegalArgumentException("Only named classes are supported."); - } - if(description instanceof Thing){ - description = new NamedClass(Thing.instance.getURI()); - } + public SortedSet<Description> getSubClasses(Description description, boolean direct) { SortedSet<Description> subClasses = new TreeSet<Description>(); String query; - if(useVirtuoso){ - query = getAllSubClassesVirtuosoQuery(description); - } else { + if(direct){ query = String.format("SELECT ?sub {?sub <%s> <%s>. FILTER(isIRI(?sub))}", RDFS.subClassOf.getURI(), - ((NamedClass)description).getURI().toString() - - ); + ((NamedClass)description).getURI().toString()); + } else { + query = String.format("SELECT ?sub {?sub <http://www.w3.org/2000/01/rdf-schema#subClassOf>* <%s>. }", + ((NamedClass)description).getURI().toString()); } ResultSet rs = executeSelectQuery(query); QuerySolution qs; @@ -1597,15 +1607,6 @@ return subClasses; } - private String getAllSubClassesVirtuosoQuery(Description description){ - String query = String.format( - "SELECT DISTINCT ?sub WHERE {"+ - "{SELECT ?sub ?o where {?sub rdfs:subClassOf ?o.}}"+ - "OPTION ( TRANSITIVE, t_distinct, t_in(?sub), t_out(?o), t_min (1), t_max (8), t_step ('step_no') as ?dist ) ."+ - "FILTER(?o = <%s>)}", description.toString()); - return query; - } - @Override public ObjectPropertyHierarchy getObjectPropertyHierarchy() { // TODO Auto-generated method stub Modified: trunk/components-core/src/main/java/org/dllearner/refinementoperators/ELDown3.java =================================================================== --- trunk/components-core/src/main/java/org/dllearner/refinementoperators/ELDown3.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/main/java/org/dllearner/refinementoperators/ELDown3.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -180,7 +180,7 @@ refinements.addAll(refineLabel(tree, v, position)); } refinements.addAll(refineEdge(tree, v, position)); - if(v.isClassNode() && v.getLevel() < 2){ + if(v.isClassNode() && v.getLevel() < 4){ refinements.addAll(attachSubtree2(tree, v, position)); refinements.addAll(attachSubtreeDatatypeProperties(tree, v, position)); } Modified: trunk/components-core/src/test/java/org/dllearner/algorithms/isle/DBpediaExperiment.java =================================================================== --- trunk/components-core/src/test/java/org/dllearner/algorithms/isle/DBpediaExperiment.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/test/java/org/dllearner/algorithms/isle/DBpediaExperiment.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -27,7 +27,9 @@ import java.util.concurrent.Executors; import org.apache.log4j.Logger; +import org.coode.owlapi.rdfxml.parser.DataSomeValuesFromTranslator; import org.dllearner.algorithms.celoe.CELOE; +import org.dllearner.algorithms.el.ELLearningAlgorithmDisjunctive; import org.dllearner.algorithms.elcopy.ELLearningAlgorithm; import org.dllearner.algorithms.isle.index.Index; import org.dllearner.algorithms.isle.index.RelevanceMapGenerator; @@ -40,6 +42,9 @@ import org.dllearner.core.ComponentInitException; import org.dllearner.core.EvaluatedDescription; import org.dllearner.core.KnowledgeSource; +import org.dllearner.core.owl.DataRange; +import org.dllearner.core.owl.Datatype; +import org.dllearner.core.owl.DatatypeSomeRestriction; import org.dllearner.core.owl.Description; import org.dllearner.core.owl.Entity; import org.dllearner.core.owl.Individual; @@ -65,9 +70,11 @@ import org.semanticweb.owlapi.model.IRI; import org.semanticweb.owlapi.model.OWLClass; import org.semanticweb.owlapi.model.OWLDataFactory; +import org.semanticweb.owlapi.model.OWLDataPropertyRangeAxiom; import org.semanticweb.owlapi.model.OWLOntology; import org.semanticweb.owlapi.model.OWLOntologyCreationException; import org.semanticweb.owlapi.model.OWLOntologyManager; +import org.semanticweb.owlapi.util.OWLEntityRemover; import com.google.common.base.Charsets; import com.google.common.collect.Sets; @@ -75,8 +82,11 @@ import com.google.common.hash.Hashing; import com.google.common.io.Files; import com.hp.hpl.jena.datatypes.xsd.XSDDatatype; +import com.hp.hpl.jena.query.Query; import com.hp.hpl.jena.query.QueryExecutionFactory; +import com.hp.hpl.jena.query.QueryFactory; import com.hp.hpl.jena.query.ResultSetFormatter; +import com.hp.hpl.jena.query.Syntax; import com.hp.hpl.jena.rdf.model.Literal; import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Property; @@ -94,12 +104,13 @@ */ public class DBpediaExperiment { + private static final Logger logger = Logger.getLogger(DBpediaExperiment.class.getName()); private DecimalFormat dfPercent = new DecimalFormat("0.00%"); HashFunction hf = Hashing.md5(); - SparqlEndpoint endpoint = SparqlEndpoint.getEndpointDBpediaLOD2Cloud(); + SparqlEndpoint endpoint = SparqlEndpoint.getEndpointDBpedia(); String namespace = "http://dbpedia.org/ontology/"; OWLOntology schema; @@ -112,14 +123,15 @@ private SPARQLReasoner reasoner; private AutomaticNegativeExampleFinderSPARQL2 negativeExampleFinder; - final int maxNrOfPositiveExamples = 20; - final int maxNrOfNegativeExamples = 50; + final int maxNrOfPositiveExamples = 100; + final int maxNrOfNegativeExamples = 200; boolean posOnly = false; int maxCBDDepth = 1; //learning algorithm settings private int maxNrOfResults = 50; private int maxExecutionTimeInSeconds = 10; + private double noiseInPercentage = 70; private boolean useNegation = false; private boolean useAllConstructor = false; @@ -149,11 +161,10 @@ } public void run(){ - ExecutorService es = Executors.newFixedThreadPool(6); - Set<NamedClass> classes = getClasses(); + Set<NamedClass> classes = getClasses(); classes = reasoner.getMostSpecificClasses(); List<NamedClass> classList = new ArrayList<>(classes); - Collections.reverse(classList); +// Collections.reverse(classList); for (NamedClass cls : classList) { try { @@ -189,7 +200,12 @@ //generate a sample of the knowledge base based on the examples OWLOntology knowledgebaseSample = loadKnowledgebaseSample(Sets.union(positiveExamples, negativeExamples)); // Map<Entity, Double> entityRelevance = RelevanceMapGenerator.generateRelevanceMap(cls, schema, relevanceMetric, true); - + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + //set up the learning try { // set KB @@ -230,6 +246,7 @@ mon.stop(); System.out.println(mon.getLastValue()); **/ + // 1. run basic algorithm //set up the refinement operator and the allowed OWL constructs RhoDRDown rop = new RhoDRDown(); @@ -243,7 +260,7 @@ AbstractCELA la; if(useEL){ la = new ELLearningAlgorithm(lp, reasoner); - ((ELLearningAlgorithm)la).setNoisePercentage(30); + ((ELLearningAlgorithm)la).setNoisePercentage(noiseInPercentage); ((ELLearningAlgorithm)la).setStartClass(startClass); ((ELLearningAlgorithm)la).setIgnoredConcepts(Sets.newHashSet(cls)); ((ELLearningAlgorithm)la).setClassToDescribe(cls); @@ -275,7 +292,7 @@ for(EvaluatedDescription ed : la.getCurrentlyBestEvaluatedDescriptions().descendingSet()) { if(lp instanceof PosNegLPStandard) { double fMeasure = ((PosNegLPStandard)lp).getFMeasureOrTooWeakExact(ed.getDescription(),1); - sb.append(ed.getDescription().toManchesterSyntaxString(reasoner.getBaseURI(), reasoner.getPrefixes()) + "," + sb.append(replaceDataPropertyRanges(ed.getDescription()).toManchesterSyntaxString(reasoner.getBaseURI(), reasoner.getPrefixes()) + "," // + ((PosNegLPStandard)lp).getPredAccuracyOrTooWeakExact(ed.getDescription(),1) + "," + fMeasure); double relevanceScore = getRelevanceScore(ed.getDescription(), entityRelevance); @@ -430,10 +447,8 @@ logger.info("Done. Size: " + sampleModel.size() + " triples"); cleanUp(sampleModel); logger.info("Clean up. Size: " + sampleModel.size() + " triples"); -// String query = "SELECT distinct ?s WHERE {?s a <http://dbpedia.org/ontology/Astronaut>. ?s <http://dbpedia.org/ontology/mission> ?o.}"; +// Query query = QueryFactory.create("SELECT ?p (COUNT(distinct ?s) AS ?cnt) WHERE {?s ?p ?o. ?s a <http://dbpedia.org/ontology/Cardinal>} GROUP BY ?p ORDER BY DESC(?cnt)", Syntax.syntaxARQ); // System.out.println(ResultSetFormatter.asText(QueryExecutionFactory.create(query, sampleModel).execSelect())); -// query = "SELECT distinct ?s WHERE {?s a <http://dbpedia.org/ontology/Astronaut>. ?s <http://dbpedia.org/ontology/timeInSpace> ?o. }"; -// System.out.println(ResultSetFormatter.asText(QueryExecutionFactory.create(query, sampleModel).execSelect())); try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -445,10 +460,19 @@ man.removeAxioms(ontology, ontology.getAxioms(AxiomType.FUNCTIONAL_DATA_PROPERTY)); man.removeAxioms(ontology, ontology.getAxioms(AxiomType.FUNCTIONAL_OBJECT_PROPERTY)); man.removeAxioms(ontology, ontology.getAxioms(AxiomType.DATA_PROPERTY_RANGE)); + man.removeAxioms(ontology, ontology.getAxioms(AxiomType.DISJOINT_CLASSES)); man.removeAxioms(ontology, ontology.getAxioms(AxiomType.SAME_INDIVIDUAL)); man.removeAxiom(ontology, df.getOWLObjectPropertyDomainAxiom( df.getOWLObjectProperty(IRI.create("http://dbpedia.org/ontology/mission")), df.getOWLClass(IRI.create("http://dbpedia.org/ontology/Aircraft")))); + OWLEntityRemover remover = new OWLEntityRemover(man, Sets.newHashSet(ontology)); + for (OWLClass cls : ontology.getClassesInSignature()) { + if(!cls.toStringID().startsWith("http://dbpedia.org/ontology/")){ + cls.accept(remover); + } + } + man.applyChanges(remover.getChanges()); + return ontology; } catch (Exception e) { e.printStackTrace(); @@ -462,7 +486,7 @@ dbo + "wikiPageDisambiguates", dbo + "wikiPageExternalLink", dbo + "wikiPageID", dbo + "wikiPageInterLanguageLink", dbo + "wikiPageRedirects", dbo + "wikiPageRevisionID", - dbo + "wikiPageWikiLink", dbo + "thumbnail", dbo + "abstract");System.out.println(blackList); + dbo + "wikiPageWikiLink", dbo + "thumbnail", dbo + "abstract"); // filter out triples with String literals, as therein often occur // some syntax errors and they are not relevant for learning List<Statement> statementsToRemove = new ArrayList<Statement>(); @@ -528,6 +552,23 @@ // } } + private Description replaceDataPropertyRanges(Description d){ + Description description = d.clone(); + List<Description> children = description.getChildren(); + for(int i=0; i < children.size(); i++) { + Description child = children.get(i); + if(child instanceof DatatypeSomeRestriction){ + Set<OWLDataPropertyRangeAxiom> rangeAxioms = schema.getDataPropertyRangeAxioms(schema.getOWLOntologyManager().getOWLDataFactory().getOWLDataProperty(IRI.create(((DatatypeSomeRestriction) child).getRole().getName()))); + if(!rangeAxioms.isEmpty()){ + DataRange range = new Datatype(rangeAxioms.iterator().next().getRange().asOWLDatatype().toStringID()); + description.replaceChild(i, new DatatypeSomeRestriction(((DatatypeSomeRestriction) child).getRole(), range)); + } + + } + } + return description; + } + private Index getSyntacticIndex(){ return new SolrSyntacticIndex(schema, solrServerURL, searchField); } Modified: trunk/components-core/src/test/java/org/dllearner/algorithms/isle/KnowledgebaseSampleGenerator.java =================================================================== --- trunk/components-core/src/test/java/org/dllearner/algorithms/isle/KnowledgebaseSampleGenerator.java 2014-01-30 08:43:49 UTC (rev 4216) +++ trunk/components-core/src/test/java/org/dllearner/algorithms/isle/KnowledgebaseSampleGenerator.java 2014-02-03 14:04:40 UTC (rev 4217) @@ -136,6 +136,11 @@ Model cbd; for (Individual individual : individuals) { try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + try { cbd = cbdGen.getConciseBoundedDescription(individual.getName(), maxCBDDepth, true); model.add(cbd); } catch (Exception e) { This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |