Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[3502] trunk/interfaces/src/main/java/org/dllearner/ cli
|
From: <lor...@us...> - 2011-12-13 11:58:47
|
Revision: 3502
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=3502&view=rev
Author: lorenz_b
Date: 2011-12-13 11:58:38 +0000 (Tue, 13 Dec 2011)
Log Message:
-----------
Made global enrichment script multi-threaded.
Modified Paths:
--------------
trunk/interfaces/src/main/java/org/dllearner/cli/Enrichment.java
trunk/interfaces/src/main/java/org/dllearner/cli/GlobalEnrichment.java
Modified: trunk/interfaces/src/main/java/org/dllearner/cli/Enrichment.java
===================================================================
--- trunk/interfaces/src/main/java/org/dllearner/cli/Enrichment.java 2011-12-13 11:57:29 UTC (rev 3501)
+++ trunk/interfaces/src/main/java/org/dllearner/cli/Enrichment.java 2011-12-13 11:58:38 UTC (rev 3502)
@@ -278,8 +278,8 @@
SPARQLTasks st = new SPARQLTasks(se);
//check if endpoint supports SPARQL 1.1
- boolean supportsSPARQL_1_1 = st.supportsSPARQL_1_1();
- ks.setSupportsSPARQL_1_1(supportsSPARQL_1_1);
+// boolean supportsSPARQL_1_1 = st.supportsSPARQL_1_1();
+// ks.setSupportsSPARQL_1_1(supportsSPARQL_1_1);
if(useInference){
reasoner = new SPARQLReasoner(ks);
Modified: trunk/interfaces/src/main/java/org/dllearner/cli/GlobalEnrichment.java
===================================================================
--- trunk/interfaces/src/main/java/org/dllearner/cli/GlobalEnrichment.java 2011-12-13 11:57:29 UTC (rev 3501)
+++ trunk/interfaces/src/main/java/org/dllearner/cli/GlobalEnrichment.java 2011-12-13 11:58:38 UTC (rev 3502)
@@ -32,9 +32,10 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
-import javax.xml.ws.http.HTTPException;
-
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -70,6 +71,15 @@
// directory for generated schemata
private static String baseDir = "log/lod-enriched/";
+
+ //parameters for thread pool
+ //Parallel running Threads(Executor) on System
+ private static int corePoolSize = 10;
+ //Maximum Threads allowed in Pool
+ private static int maximumPoolSize = 20;
+ //Keep alive time for waiting threads for jobs(Runnable)
+ private static long keepAliveTime = 10;
+
/**
* @param args
* @throws MalformedURLException
@@ -129,58 +139,91 @@
TreeSet<String> blacklist = new TreeSet<String>();
blacklist.add("rkb-explorer-crime"); // computation never completes
+ ArrayBlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<Runnable>(endpoints.size());
+ ThreadPoolExecutor threadPool = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.SECONDS, workQueue);
+
+
// perform enrichment on endpoints
- for(Entry<String,SparqlEndpoint> endpoint : endpoints.entrySet()) {
- // run enrichment
- SparqlEndpoint se = endpoint.getValue();
- String name = endpoint.getKey();
+ for(final Entry<String,SparqlEndpoint> endpoint : endpoints.entrySet()) {
- File f = new File(baseDir + name + ".ttl");
- File log = new File(baseDir + name + ".log");
+ threadPool.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ // run enrichment
+ SparqlEndpoint se = endpoint.getValue();
+ String name = endpoint.getKey();
+
+ File f = new File(baseDir + name + ".ttl");
+ File log = new File(baseDir + name + ".log");
+
+ System.out.println("Enriching " + name + " using " + se.getURL());
+ Enrichment e = new Enrichment(se, null, threshold, nrOfAxiomsToLearn, useInference, false);
+
+ e.maxEntitiesPerType = 3; // hack for faster testing of endpoints
+
+// if(blacklist.contains(name)) {
+// continue;
+// }
+
+ boolean success = false;
+ // run enrichment script - we make a case distinguish to see which kind of problems we get
+ // (could be interesting for statistics later on)
+ try {
+ e.start();
+ success = true;
+ } catch(StackOverflowError error) {
+ try {
+ error.printStackTrace(new PrintStream(log));
+ } catch (FileNotFoundException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+ Files.appendToFile(log, "stack overflows could be caused by cycles in class hierarchies");
+ error.printStackTrace();
+ } catch(ResultSetException ex) {
+ try {
+ ex.printStackTrace(new PrintStream(log));
+ } catch (FileNotFoundException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+ Files.appendToFile(log, ex.getMessage());
+ ex.printStackTrace();
+ } catch(QueryExceptionHTTP ex) {
+ try {
+ ex.printStackTrace(new PrintStream(log));
+ } catch (FileNotFoundException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+ Files.appendToFile(log, ex.getMessage());
+ ex.printStackTrace();
+ }
+ catch(Exception ex) {
+ System.out.println("class of exception: " + ex.getClass());
+ }
+
+ // save results to a file (TODO: check if enrichment format
+ if(success) {
+ SparqlEndpointKS ks = new SparqlEndpointKS(se);
+ List<AlgorithmRun> runs = e.getAlgorithmRuns();
+ List<OWLAxiom> axioms = new LinkedList<OWLAxiom>();
+ for(AlgorithmRun run : runs) {
+ axioms.addAll(e.toRDF(run.getAxioms(), run.getAlgorithm(), run.getParameters(), ks));
+ }
+ Model model = e.getModel(axioms);
+ try {
+ model.write(new FileOutputStream(f), "TURTLE");
+ } catch (FileNotFoundException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+ }
+
+ }
+ });
- System.out.println("Enriching " + name + " using " + se);
- Enrichment e = new Enrichment(se, null, threshold, nrOfAxiomsToLearn, useInference, false);
-
- e.maxEntitiesPerType = 3; // hack for faster testing of endpoints
-
- if(blacklist.contains(name)) {
- continue;
- }
-
- boolean success = false;
- // run enrichment script - we make a case distinguish to see which kind of problems we get
- // (could be interesting for statistics later on)
- try {
- e.start();
- success = true;
- } catch(StackOverflowError error) {
- error.printStackTrace(new PrintStream(log));
- Files.appendToFile(log, "stack overflows could be caused by cycles in class hierarchies");
- error.printStackTrace();
- } catch(ResultSetException ex) {
- ex.printStackTrace(new PrintStream(log));
- Files.appendToFile(log, ex.getMessage());
- ex.printStackTrace();
- } catch(QueryExceptionHTTP ex) {
- ex.printStackTrace(new PrintStream(log));
- Files.appendToFile(log, ex.getMessage());
- ex.printStackTrace();
- }
-// catch(Exception ex) {
-// System.out.println("class of exception: " + ex.getClass());
-// }
-
- // save results to a file (TODO: check if enrichment format
- if(success) {
- SparqlEndpointKS ks = new SparqlEndpointKS(se);
- List<AlgorithmRun> runs = e.getAlgorithmRuns();
- List<OWLAxiom> axioms = new LinkedList<OWLAxiom>();
- for(AlgorithmRun run : runs) {
- axioms.addAll(e.toRDF(run.getAxioms(), run.getAlgorithm(), run.getParameters(), ks));
- }
- Model model = e.getModel(axioms);
- model.write(new FileOutputStream(f), "TURTLE");
- }
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
