Menu
▾
▴
bigdata-commit
|
From: <tho...@us...> - 2010-12-19 19:09:34
|
Revision: 4015
http://bigdata.svn.sourceforge.net/bigdata/?rev=4015&view=rev
Author: thompsonbry
Date: 2010-12-19 19:09:28 +0000 (Sun, 19 Dec 2010)
Log Message:
-----------
Modified the DataLoader to pass through the overridden properties with new DataLoader(properties,kb) rather than using AbstractTripleStore.getDataLoader(), which uses the defaults from the properties associated with the kb.
Bug fix to incremental TM option.
Modified to permit operations when no source files are specified (e.g., just do the closure).
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-12-19 19:05:23 UTC (rev 4014)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-12-19 19:09:28 UTC (rev 4015)
@@ -860,7 +860,7 @@
// rdfFormat);
loadFiles(totals, depth + 1, f, baseURI, rdfFormat, defaultGraph, filter,
- (depth == 0 && i < files.length ? false : endOfBatch));
+ (depth == 0 && i < (files.length-1) ? false : endOfBatch));
}
@@ -1128,12 +1128,6 @@
*/
public ClosureStats doClosure() {
- if (buffer == null)
- throw new IllegalStateException();
-
- // flush anything in the buffer.
- buffer.flush();
-
final ClosureStats stats;
switch (closureEnum) {
@@ -1145,6 +1139,12 @@
* Incremental truth maintenance.
*/
+ if (buffer == null)
+ throw new IllegalStateException();
+
+ // flush anything in the buffer.
+ buffer.flush();
+
stats = new TruthMaintenance(inferenceEngine)
.assertAll((TempTripleStore) buffer.getStatementStore());
@@ -1252,7 +1252,7 @@
final int remaining = args.length - i;
- if (remaining < 2) {
+ if (remaining < 1/*allow run w/o any named files or directories*/) {
System.err.println("Not enough arguments.");
@@ -1360,7 +1360,8 @@
}
final LoadStats totals = new LoadStats();
- final DataLoader dataLoader = kb.getDataLoader();
+ final DataLoader dataLoader = //kb.getDataLoader();
+ new DataLoader(properties,kb); // use the override properties.
for (File fileOrDir : files) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-13 14:27:34
|
Revision: 4080
http://bigdata.svn.sourceforge.net/bigdata/?rev=4080&view=rev
Author: thompsonbry
Date: 2011-01-13 14:27:28 +0000 (Thu, 13 Jan 2011)
Log Message:
-----------
Applied changes to the JOURNAL_HA_BRANCH to implement a "-quiet" option which will suppress everything on stdout.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2011-01-12 20:49:29 UTC (rev 4079)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2011-01-13 14:27:28 UTC (rev 4080)
@@ -1191,8 +1191,24 @@
* support multiple data files within a single archive.
*
* @param args
- * [-closure][-verbose][-namespace <i>namespace</i>] propertyFile (fileOrDir)+
- *
+ * <code>[-quiet][-closure][-verbose][-namespace <i>namespace</i>] propertyFile (fileOrDir)*</code>
+ * where
+ * <dl>
+ * <dt>-quiet</dt>
+ * <dd>Suppress all stdout messages.</dd>
+ * <dt>-verbose</dt>
+ * <dd>Show additional messages detailing the load performance.</dd>
+ * <dt>-closure</dt>
+ * <dd>Compute the RDF(S)+ closure.</dd>
+ * <dt>-namespace</dt>
+ * <dd>The namespace of the KB instance.</dd>
+ * <dt>propertyFile</dt>
+ * <dd>The configuration file for the database instance.</dd>
+ * <dt>fileOrDir</dt>
+ * <dd>Zero or more files or directories containing the data to
+ * be loaded.</dd>
+ * </dl>
+ *
* @throws IOException
*/
public static void main(final String[] args) throws IOException {
@@ -1201,6 +1217,7 @@
String namespace = "kb";
boolean doClosure = false;
boolean verbose = false;
+ boolean quiet = false;
RDFFormat rdfFormat = null;
String baseURI = null;
@@ -1231,7 +1248,13 @@
} else if (arg.equals("-verbose")) {
verbose = true;
+ quiet = false;
+ } else if (arg.equals("-quiet")) {
+
+ quiet = true;
+ verbose = false;
+
} else {
System.err.println("Unknown argument: " + arg);
@@ -1270,7 +1293,8 @@
final Properties properties = new Properties();
{
- System.out.println("Reading properties: "+propertyFile);
+ if(!quiet)
+ System.out.println("Reading properties: "+propertyFile);
final InputStream is = new FileInputStream(propertyFile);
try {
properties.load(is);
@@ -1283,7 +1307,7 @@
// // Override/set from the environment.
// final String file = System
// .getProperty(com.bigdata.journal.Options.FILE);
-// System.out.println("Using: " + com.bigdata.journal.Options.FILE
+// if(!quiet) System.out.println("Using: " + com.bigdata.journal.Options.FILE
// + "=" + file);
// properties.setProperty(com.bigdata.journal.Options.FILE, file);
// }
@@ -1311,7 +1335,8 @@
if (System.getProperty(s) != null) {
// Override/set from the environment.
final String v = System.getProperty(s);
- System.out.println("Using: " + s + "=" + v);
+ if(!quiet)
+ System.out.println("Using: " + s + "=" + v);
properties.setProperty(s, v);
}
}
@@ -1330,7 +1355,8 @@
files.add(fileOrDir);
- System.out.println("Will load from: " + fileOrDir);
+ if(!quiet)
+ System.out.println("Will load from: " + fileOrDir);
}
@@ -1345,7 +1371,8 @@
// final long firstOffset = jnl.getRootBlockView().getNextOffset();
final long userData0 = jnl.getBufferStrategy().size();
- System.out.println("Journal file: "+jnl.getFile());
+ if(!quiet)
+ System.out.println("Journal file: "+jnl.getFile());
AbstractTripleStore kb = (AbstractTripleStore) jnl
.getResourceLocator().locate(namespace, ITx.UNISOLATED);
@@ -1376,7 +1403,8 @@
dataLoader.endSource();
- System.out.println("Load: " + totals);
+ if(!quiet)
+ System.out.println("Load: " + totals);
if (dataLoader.closureEnum == ClosureEnum.None && doClosure) {
@@ -1390,11 +1418,16 @@
new StringBuilder()).toString());
}
- System.out.println("Computing closure.");
+ if(!quiet)
+ System.out.println("Computing closure.");
+ log.info("Computing closure");
final ClosureStats stats = dataLoader.doClosure();
- System.out.println("Closure: "+stats.toString());
+ if(!quiet)
+ System.out.println("Closure: "+stats.toString());
+ if(log.isInfoEnabled())
+ log.info("Closure: "+stats.toString());
}
@@ -1428,12 +1461,16 @@
// #of bytes written (user data only)
final long bytesWritten = (userData1 - userData0);
- System.out.println("Wrote: " + bytesWritten + " bytes.");
+ if(!quiet)
+ System.out.println("Wrote: " + bytesWritten + " bytes.");
final long elapsedTotal = System.currentTimeMillis() - begin;
+
+ if (!quiet)
+ System.out.println("Total elapsed=" + elapsedTotal + "ms");
+ if (log.isInfoEnabled())
+ log.info("Total elapsed=" + elapsedTotal + "ms");
- System.out.println("Total elapsed=" + elapsedTotal + "ms");
-
} finally {
if (jnl != null) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-13 15:29:20
|
Revision: 4082
http://bigdata.svn.sourceforge.net/bigdata/?rev=4082&view=rev
Author: thompsonbry
Date: 2011-01-13 15:29:14 +0000 (Thu, 13 Jan 2011)
Log Message:
-----------
Made loadData3(...) public.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2011-01-13 15:29:00 UTC (rev 4081)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2011-01-13 15:29:14 UTC (rev 4082)
@@ -941,14 +941,26 @@
* Loads data from the <i>source</i>. The caller is responsible for closing
* the <i>source</i> if there is an error.
*
+ * @param totals
+ * Used to report out the total {@link LoadStats}.
* @param source
* A {@link Reader} or {@link InputStream}.
* @param baseURL
+ * The baseURI (optional, when not specified the name of the each
+ * file load is converted to a URL and used as the baseURI for
+ * that file).
* @param rdfFormat
+ * The format of the file (optional, when not specified the
+ * format is deduced for each file in turn using the
+ * {@link RDFFormat} static methods).
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate
+ * when loading data represented in a triple format into a quad
+ * store.
* @param endOfBatch
- * @return
+ * Signal indicates the end of a batch.
*/
- protected void loadData3(final LoadStats totals, final Object source,
+ public void loadData3(final LoadStats totals, final Object source,
final String baseURL, final RDFFormat rdfFormat,
final String defaultGraph, final boolean endOfBatch) throws IOException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X