Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2014-07-16 16:05:43
|
Revision: 8559
http://sourceforge.net/p/bigdata/code/8559
Author: thompsonbry
Date: 2014-07-16 16:05:36 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
code cleanup
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IIndexManager.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IIndexManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IIndexManager.java 2014-07-16 16:01:00 UTC (rev 8558)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IIndexManager.java 2014-07-16 16:05:36 UTC (rev 8559)
@@ -34,7 +34,6 @@
* Interface for managing named indices.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface IIndexManager extends IIndexStore, ICounterSetAccess {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java 2014-07-16 16:01:00 UTC (rev 8558)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java 2014-07-16 16:05:36 UTC (rev 8559)
@@ -29,7 +29,6 @@
import java.util.UUID;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.ILocalBTreeView;
@@ -130,9 +129,9 @@
* those that are not spanned by the namespace.
*/
- final Future<Void> ft = journal
- .submit(new AbstractTask<Void>(journal
- .getConcurrencyManager(), ITx.UNISOLATED, namespace) {
+ journal.submit(
+ new AbstractTask<Void>(journal.getConcurrencyManager(),
+ ITx.UNISOLATED, namespace) {
@Override
protected Void doTask() throws Exception {
@@ -167,15 +166,8 @@
// Done.
return null;
}
- });
+ }).get();
- try {
- // Await outcome.
- ft.get();
- } finally {
- ft.cancel(true/* mayInterruptIfRunning */);
- }
-
} finally {
journal.destroy();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-07-16 16:01:05
|
Revision: 8558
http://sourceforge.net/p/bigdata/code/8558
Author: mrpersonick
Date: 2014-07-16 16:01:00 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
Ticket #714: Sesame 2.7 migration
Modified Paths:
--------------
branches/SESAME_2_7/bigdata/src/resources/logging/log4j.properties
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/ConcatBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DigestBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IriBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsNumericBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NumericBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrAfterBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBeforeBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrdtBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrlangBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/AbstractIV.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/rio/turtle/BigdataTurtleParser.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionRegistry.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/GroupMemberValueExpressionNodeBase.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryBase.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryRoot.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTBottomUpOptimizer.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTGraphGroupOptimizer.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTJoinOrderByTypeOptimizer.java
branches/SESAME_2_7/bigdata-rdf/src/resources/service-providers/META-INF/services/org.openrdf.query.resultio.TupleQueryResultWriterFactory
branches/SESAME_2_7/bigdata-rdf/src/resources/service-providers/META-INF/services/org.openrdf.rio.RDFParserFactory
branches/SESAME_2_7/bigdata-rdf/src/resources/service-providers/META-INF/services/org.openrdf.rio.RDFWriterFactory
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailUpdate.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataValueReplacer.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ASTVisitorBase.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BaseDeclProcessor.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BigdataExprBuilder.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BlankNodeVarProcessor.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPatternBuilder.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/PrefixDeclProcessor.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/UpdateExprBuilder.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ValueExprBuilder.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilder.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderConstants.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderTokenManager.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderTreeConstants.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderVisitor.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/sparql.jj
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/sparql.jjt
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataComplexSparqlQueryTest.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataConnectionTest.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateConformanceTest.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataStoreTest.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLUpdateTest.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/query/parser/sparql/manifest/SPARQLQueryTest.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/repository/RepositoryConnectionTest.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/sail/RDFStoreTest.java
Added Paths:
-----------
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NowBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/UUIDBOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTValuesOptimizer.java
branches/SESAME_2_7/bigdata-rdf/src/resources/service-providers/META-INF/services/org.openrdf.query.resultio.TupleQueryResultParserFactory
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/rio/Bar.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/rio/Foo.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTInlineData.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTSTRUUID.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTUUID.java
branches/SESAME_2_7/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTUnparsedQuadDataBlock.java
branches/SESAME_2_7/bigdata-sails/src/java/org/openrdf/query/parser/sparql/
branches/SESAME_2_7/bigdata-sails/src/java/org/openrdf/query/parser/sparql/manifest/
branches/SESAME_2_7/bigdata-sails/src/java/org/openrdf/query/parser/sparql/manifest/SPARQL11ManifestTest.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/model/
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/model/util/
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/model/util/ModelUtil.java
branches/SESAME_2_7/bigdata-sails/src/test/org/openrdf/query/parser/sparql/manifest/SPARQLUpdateConformanceTest.java
Modified: branches/SESAME_2_7/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/SESAME_2_7/bigdata/src/resources/logging/log4j.properties 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata/src/resources/logging/log4j.properties 2014-07-16 16:01:00 UTC (rev 8558)
@@ -16,7 +16,8 @@
log4j.logger.com.bigdata.rdf.store.DataLoader=INFO
log4j.logger.com.bigdata.resources.AsynchronousOverflowTask=INFO
-#log4j.logger.com.bigdata.rdf.rio.StatementBuffer=ALL
+log4j.logger.com.bigdata.rdf.sparql=ALL
+#log4j.logger.com.bigdata.rdf.sail.sparql.BigdataExprBuilder=INFO
#log4j.logger.com.bigdata.rdf.sail.TestProvenanceQuery=ALL
#log4j.logger.com.bigdata.rdf.sail.TestSids=ALL
#log4j.logger.com.bigdata.rdf.sail.ProxyBigdataSailTestCase=ALL
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/ConcatBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/ConcatBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/ConcatBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -31,6 +31,7 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
+import com.bigdata.rdf.error.SparqlTypeErrorException;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
@@ -76,6 +77,11 @@
for (int i = 0; i < arity(); i++) {
@SuppressWarnings("rawtypes")
final IV v = getAndCheckLiteral(i, bs);
+
+ if (v.isNumeric()) {
+ throw new SparqlTypeErrorException();
+ }
+
String label = null;
if (allSame) {
final Literal lit = asLiteral(v);
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -30,6 +30,7 @@
import org.openrdf.model.Literal;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
+import org.openrdf.model.vocabulary.RDF;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
@@ -121,6 +122,11 @@
// literal with datatype
datatype = literal.getDatatype();
+ } else if (literal.getLanguage() != null) {
+
+ // language-tag literal
+ datatype = RDF.LANGSTRING;
+
} else if (literal.getLanguage() == null) {
// simple literal
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -23,10 +23,17 @@
*/
package com.bigdata.rdf.internal.constraints;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.util.Map;
import javax.xml.datatype.XMLGregorianCalendar;
+import org.openrdf.model.Literal;
+import org.openrdf.query.algebra.evaluation.ValueExprEvaluationException;
+import org.openrdf.query.algebra.evaluation.function.datetime.Timezone;
+import org.openrdf.query.algebra.evaluation.function.datetime.Tz;
+
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
@@ -36,8 +43,12 @@
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.NotMaterializedException;
import com.bigdata.rdf.internal.XSD;
-import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
+import com.bigdata.rdf.internal.impl.literal.XSDDecimalIV;
+import com.bigdata.rdf.internal.impl.literal.XSDIntegerIV;
import com.bigdata.rdf.model.BigdataLiteral;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.sparql.ast.DummyConstantNode;
+import com.bigdata.rdf.sparql.ast.FilterNode;
import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
/**
@@ -63,14 +74,14 @@
}
public enum DateOp {
- YEAR, MONTH, DAY, HOURS, MINUTES, SECONDS, TIMEZONE;
+ YEAR, MONTH, DAY, HOURS, MINUTES, SECONDS, TZ, TIMEZONE
}
@Override
protected boolean areGlobalsRequired() {
- return false;
+ return true;
}
@@ -84,9 +95,9 @@
* The annotation specifying the operation to be performed on those operands.
*/
public DateBOp(final IValueExpression<? extends IV> left,
- final DateOp op) {
+ final DateOp op, final GlobalAnnotations globals) {
- this(new BOp[] { left }, NV.asMap(Annotations.OP, op));
+ this(new BOp[] { left }, anns(globals, new NV(Annotations.OP, op)));
}
@@ -140,19 +151,21 @@
XMLGregorianCalendar cal=bl.calendarValue();
switch (op()) {
case DAY:
- return new XSDNumericIV(cal.getDay());
+ return new XSDIntegerIV(BigInteger.valueOf(cal.getDay()));
case MONTH:
- return new XSDNumericIV(cal.getMonth());
+ return new XSDIntegerIV(BigInteger.valueOf(cal.getMonth()));
case YEAR:
- return new XSDNumericIV(cal.getYear());
+ return new XSDIntegerIV(BigInteger.valueOf(cal.getYear()));
case HOURS:
- return new XSDNumericIV(cal.getHour());
+ return new XSDIntegerIV(BigInteger.valueOf(cal.getHour()));
case SECONDS:
- return new XSDNumericIV(cal.getSecond());
+ return new XSDDecimalIV(BigDecimal.valueOf(cal.getSecond()));
case MINUTES:
- return new XSDNumericIV(cal.getMinute());
+ return new XSDIntegerIV(BigInteger.valueOf(cal.getMinute()));
+ case TZ:
+ return tz(bl);
case TIMEZONE:
- return new XSDNumericIV(cal.getTimezone());
+ return timezone(bl);
default:
throw new UnsupportedOperationException();
}
@@ -181,5 +194,45 @@
public Requirement getRequirement() {
return Requirement.SOMETIMES;
}
+
+ protected IV tz(final BigdataLiteral l) {
+
+ final Tz func = new Tz();
+
+ final BigdataValueFactory vf = super.getValueFactory();
+
+ try {
+
+ final BigdataLiteral l2 = (BigdataLiteral) func.evaluate(vf, l);
+
+ return DummyConstantNode.toDummyIV(l2);
+
+ } catch (ValueExprEvaluationException e) {
+
+ throw new SparqlTypeErrorException();
+
+ }
+
+ }
+ protected IV timezone(final BigdataLiteral l) {
+
+ final Timezone func = new Timezone();
+
+ final BigdataValueFactory vf = super.getValueFactory();
+
+ try {
+
+ final BigdataLiteral l2 = (BigdataLiteral) func.evaluate(vf, l);
+
+ return DummyConstantNode.toDummyIV(l2);
+
+ } catch (ValueExprEvaluationException e) {
+
+ throw new SparqlTypeErrorException();
+
+ }
+
+ }
+
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DigestBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DigestBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DigestBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -146,8 +146,9 @@
//Recreate since they are not thread safe
MessageDigest md = null;
final Literal lit = asLiteral(iv);
- if (lit.getLanguage() != null || lit.getDatatype() != null
- && lit.getDatatype().equals(XSD.STRING)) {
+ if (lit.getLanguage() == null &&
+ (lit.getDatatype() == null || lit.getDatatype().equals(XSD.STRING))) {
+
try {
String label = lit.getLabel();
switch (op()) {
@@ -155,19 +156,19 @@
md = MessageDigest.getInstance("MD5");
break;
case SHA1:
- md = MessageDigest.getInstance("SHA1");
+ md = MessageDigest.getInstance("SHA-1");
break;
case SHA224:
- md = MessageDigest.getInstance("SHA224");
+ md = MessageDigest.getInstance("SHA-224");
break;
case SHA256:
- md = MessageDigest.getInstance("SHA256");
+ md = MessageDigest.getInstance("SHA-256");
break;
case SHA384:
- md = MessageDigest.getInstance("SHA384");
+ md = MessageDigest.getInstance("SHA-384");
break;
case SHA512:
- md = MessageDigest.getInstance("SHA512");
+ md = MessageDigest.getInstance("SHA-512");
break;
default:
throw new UnsupportedOperationException();
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IriBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IriBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IriBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -27,10 +27,13 @@
import org.openrdf.model.Literal;
import org.openrdf.model.URI;
+import org.openrdf.query.algebra.evaluation.ValueExprEvaluationException;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
+import com.bigdata.bop.ImmutableBOp;
+import com.bigdata.bop.NV;
import com.bigdata.rdf.error.SparqlTypeErrorException;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.XSD;
@@ -49,9 +52,17 @@
private static final long serialVersionUID = -8448763718374010166L;
+
+ public interface Annotations extends ImmutableBOp.Annotations {
+
+ String BASE_URI = (IriBOp.class.getName() + ".baseURI").intern();
+
+ }
+
public IriBOp(final IValueExpression<? extends IV> x,
- final GlobalAnnotations globals) {
- super(x, globals);
+ final String baseURI,
+ final GlobalAnnotations globals) {
+ super(x, globals, new NV(Annotations.BASE_URI, baseURI));
}
public IriBOp(BOp[] args, Map<String, Object> anns) {
@@ -81,6 +92,8 @@
if (!iv.isLiteral())
throw new SparqlTypeErrorException();
+ final String baseURI = getProperty(Annotations.BASE_URI, "");
+
final Literal lit = asLiteral(iv);
final URI dt = lit.getDatatype();
@@ -88,8 +101,21 @@
if (dt != null && !dt.stringValue().equals(XSD.STRING.stringValue()))
throw new SparqlTypeErrorException();
- final BigdataURI uri = getValueFactory().createURI(lit.getLabel());
+// final BigdataURI uri = getValueFactory().createURI(baseURI+lit.getLabel());
+ BigdataURI uri = null;
+ try {
+ uri = getValueFactory().createURI(lit.getLabel());
+ }
+ catch (IllegalArgumentException e) {
+ try {
+ uri = getValueFactory().createURI(baseURI, lit.getLabel());
+ }
+ catch (IllegalArgumentException e1) {
+ throw new SparqlTypeErrorException();
+ }
+ }
+
return super.asIV(uri, bs);
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsNumericBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsNumericBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsNumericBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -76,7 +76,7 @@
if (iv == null)
throw new SparqlTypeErrorException.UnboundVarException();
- return iv.isNumeric();
+ return iv.isNumeric() && !iv.isExtension();
}
Added: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NowBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NowBOp.java (rev 0)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NowBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -0,0 +1,112 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package com.bigdata.rdf.internal.constraints;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.Map;
+
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.ImmutableBOp;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.sparql.ast.FilterNode;
+import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
+
+/**
+ * Implements the now() operator.
+ */
+public class NowBOp extends IVValueExpression<IV> implements INeedsMaterialization{
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 9136864442064392445L;
+
+ public interface Annotations extends ImmutableBOp.Annotations {
+
+ }
+
+ public NowBOp(final GlobalAnnotations globals) {
+
+ this(BOp.NOARGS, anns(globals));
+
+ }
+
+ /**
+ * Required shallow copy constructor.
+ *
+ * @param args
+ * The operands.
+ * @param op
+ * The operation.
+ */
+ public NowBOp(final BOp[] args, Map<String, Object> anns) {
+
+ super(args, anns);
+
+ }
+
+ /**
+ * Constructor required for {@link com.bigdata.bop.BOpUtility#deepCopy(FilterNode)}.
+ *
+ * @param op
+ */
+ public NowBOp(final NowBOp op) {
+
+ super(op);
+
+ }
+
+ final public IV get(final IBindingSet bs) {
+
+ final Calendar cal = Calendar.getInstance();
+ final Date now = cal.getTime();
+ final GregorianCalendar c = new GregorianCalendar();
+ c.setTime(now);
+ try {
+ final XMLGregorianCalendar date =
+ DatatypeFactory.newInstance().newXMLGregorianCalendar(c);
+ return super.asIV(getValueFactory().createLiteral(date), bs);
+ } catch (DatatypeConfigurationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String toString() {
+
+ return "now()";
+
+ }
+
+ public Requirement getRequirement() {
+ return Requirement.NEVER;
+ }
+
+}
Property changes on: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NowBOp.java
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NumericBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NumericBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NumericBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -27,6 +27,8 @@
import org.apache.log4j.Logger;
import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.vocabulary.XMLSchema;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
@@ -34,6 +36,7 @@
import com.bigdata.bop.ImmutableBOp;
import com.bigdata.bop.NV;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.sparql.ast.FilterNode;
/**
* A math expression involving a left and right IValueExpression operand. The
@@ -124,6 +127,17 @@
final Literal lit = super.getAndCheckLiteralValue(0, bs);
+ final URI dt = lit.getDatatype();
+
+ final NumericOp op = op();
+
+ if ((dt.equals(XMLSchema.INT) || dt.equals(XMLSchema.INTEGER)) &&
+ (op == NumericOp.CEIL || op == NumericOp.FLOOR || op == NumericOp.ROUND)) {
+
+ return get(0).get(bs);
+
+ }
+
if (log.isDebugEnabled())
log.debug(lit);
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrAfterBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrAfterBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrAfterBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -94,7 +94,7 @@
if (s2.isEmpty()) {
- return ret(arg1, "", bs);
+ return ret(arg1, arg1.getLabel(), bs);
}
@@ -105,7 +105,7 @@
// didn't find it
if (i < 0) {
- return ret(arg1, "", bs);
+ return ret(arg1, null, bs);
}
@@ -124,6 +124,14 @@
private IV ret(final Literal arg1, final String label, final IBindingSet bs) {
+ if (label == null) {
+
+ final BigdataLiteral str = getValueFactory().createLiteral("");
+
+ return super.asIV(str, bs);
+
+ }
+
final String lang = arg1.getLanguage();
if (lang != null) {
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBeforeBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBeforeBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBeforeBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -102,7 +102,7 @@
// didn't find it
if (i < 0) {
- return ret(arg1, "", bs);
+ return ret(arg1, null, bs);
}
@@ -120,7 +120,15 @@
}
private IV ret(final Literal arg1, final String label, final IBindingSet bs) {
-
+
+ if (label == null) {
+
+ final BigdataLiteral str = getValueFactory().createLiteral("");
+
+ return super.asIV(str, bs);
+
+ }
+
final String lang = arg1.getLanguage();
if (lang != null) {
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrdtBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrdtBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrdtBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -78,6 +78,10 @@
final Literal lit = asLiteral(iv);
+ if (lit.getDatatype() != null || lit.getLanguage() != null) {
+ throw new SparqlTypeErrorException();
+ }
+
final String label = lit.getLabel();
final BigdataLiteral str = getValueFactory().createLiteral(label, dt);
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrlangBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrlangBOp.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrlangBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -67,6 +67,11 @@
public IV get(final IBindingSet bs) throws SparqlTypeErrorException {
final Literal lit = getAndCheckLiteralValue(0, bs);
+
+ if (lit.getDatatype() != null || lit.getLanguage() != null) {
+ throw new SparqlTypeErrorException();
+ }
+
final Literal l = getAndCheckLiteralValue(1, bs);
String label = lit.getLabel();
String langLit = l.getLabel();
Added: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/UUIDBOp.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/UUIDBOp.java (rev 0)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/UUIDBOp.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -0,0 +1,133 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package com.bigdata.rdf.internal.constraints;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.Map;
+import java.util.UUID;
+
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.ImmutableBOp;
+import com.bigdata.bop.NV;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.model.BigdataLiteral;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.sparql.ast.DummyConstantNode;
+import com.bigdata.rdf.sparql.ast.FilterNode;
+import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
+
+/**
+ * Implements the now() operator.
+ */
+public class UUIDBOp extends IVValueExpression<IV> implements INeedsMaterialization{
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 9136864442064392445L;
+
+ public interface Annotations extends ImmutableBOp.Annotations {
+
+ String STR = UUIDBOp.class.getName() + ".str";
+
+ }
+
+ public UUIDBOp(final GlobalAnnotations globals, final boolean str) {
+
+ this(BOp.NOARGS, anns(globals, new NV(Annotations.STR, str)));
+
+ }
+
+ /**
+ * Required shallow copy constructor.
+ *
+ * @param args
+ * The operands.
+ * @param op
+ * The operation.
+ */
+ public UUIDBOp(final BOp[] args, Map<String, Object> anns) {
+
+ super(args, anns);
+
+ }
+
+ /**
+ * Constructor required for {@link com.bigdata.bop.BOpUtility#deepCopy(FilterNode)}.
+ *
+ * @param op
+ */
+ public UUIDBOp(final UUIDBOp op) {
+
+ super(op);
+
+ }
+
+ final public IV get(final IBindingSet bs) {
+
+ final BigdataValueFactory vf = super.getValueFactory();
+
+ final UUID uuid = UUID.randomUUID();
+
+ if (str()) {
+
+ final BigdataLiteral l = vf.createLiteral(uuid.toString());
+
+ return DummyConstantNode.toDummyIV(l);
+
+ } else {
+
+ final BigdataURI uri = vf.createURI("urn:uuid:" + uuid.toString());
+
+ return DummyConstantNode.toDummyIV(uri);
+
+ }
+
+ }
+
+ public boolean str() {
+
+ return (boolean) getProperty(Annotations.STR);
+
+ }
+
+ public String toString() {
+
+ return str() ? "struuid()" : "uuid()";
+
+ }
+
+ public Requirement getRequirement() {
+ return Requirement.NEVER;
+ }
+
+}
Property changes on: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/UUIDBOp.java
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/AbstractIV.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/AbstractIV.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/AbstractIV.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -527,7 +527,7 @@
return false;
}
- final public boolean isNumeric() {
+ public boolean isNumeric() {
return isInline() && getDTE().isNumeric();
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -176,7 +176,9 @@
&& ((BigdataValue)o).isRealIV()
&& ((BigdataValue) o).getValueFactory() == getValueFactory()) {
- return getIV().equals(((BigdataValue) o).getIV());
+ final boolean equals = getIV().equals(((BigdataValue) o).getIV());
+
+ return equals;
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/rio/turtle/BigdataTurtleParser.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/rio/turtle/BigdataTurtleParser.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/rio/turtle/BigdataTurtleParser.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -15,6 +15,7 @@
import org.openrdf.model.Value;
import org.openrdf.rio.RDFHandlerException;
import org.openrdf.rio.RDFParseException;
+import org.openrdf.rio.helpers.BasicParserSettings;
import org.openrdf.rio.turtle.TurtleParser;
import org.openrdf.rio.turtle.TurtleUtil;
@@ -150,6 +151,28 @@
}
/**
+ * Consumes any white space characters (space, tab, line feed, newline) and
+ * comments (#-style) from <tt>reader</tt>. After this method has been
+ * called, the first character that is returned by <tt>reader</tt> is either
+ * a non-ignorable character, or EOF. For convenience, this character is also
+ * returned by this method.
+ *
+ * @return The next character that will be returned by <tt>reader</tt>.
+ */
+ protected int skipWS()
+ throws IOException
+ {
+ int c = read();
+ while (TurtleUtil.isWhitespace(c)) {
+ c = read();
+ }
+
+ unread(c);
+
+ return c;
+ }
+
+ /**
* Parses a blank node ID, e.g. <tt>_:node1</tt>.
*/
protected BNode parseNodeID()
@@ -164,46 +187,39 @@
if (c == -1) {
throwEOFException();
}
-// modified to allow fully numeric bnode ids
-// else if (!TurtleUtil.isNameStartChar(c)) {
-// reportError("Expected a letter, found '" + (char)c + "'");
-// }
+// modified to allow fully numeric bnode ids
+// else if (!TurtleUtil.isNameStartChar(c)) {
+// reportError("Expected a letter, found '" + (char)c + "'", BasicParserSettings.PRESERVE_BNODE_IDS);
+// }
StringBuilder name = new StringBuilder(32);
name.append((char)c);
// Read all following letter and numbers, they are part of the name
c = read();
+
+ // If we would never go into the loop we must unread now
+ if (!TurtleUtil.isNameChar(c)) {
+ unread(c);
+ }
+
while (TurtleUtil.isNameChar(c)) {
- name.append((char)c);
+ int previous = c;
c = read();
+
+ if (previous == '.' && (c == -1 || TurtleUtil.isWhitespace(c) || c == '<' || c == '_')) {
+ unread(c);
+ unread(previous);
+ break;
+ }
+ name.append((char)previous);
+ if(!TurtleUtil.isNameChar(c))
+ {
+ unread(c);
+ }
}
- unread(c);
-
return createBNode(name.toString());
}
- /**
- * Consumes any white space characters (space, tab, line feed, newline) and
- * comments (#-style) from <tt>reader</tt>. After this method has been
- * called, the first character that is returned by <tt>reader</tt> is either
- * a non-ignorable character, or EOF. For convenience, this character is also
- * returned by this method.
- *
- * @return The next character that will be returned by <tt>reader</tt>.
- */
- protected int skipWS()
- throws IOException
- {
- int c = read();
- while (TurtleUtil.isWhitespace(c)) {
- c = read();
- }
-
- unread(c);
-
- return c;
- }
-
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -42,7 +42,8 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public class BindingsClause extends ASTBase {
+public class BindingsClause extends GroupMemberNodeBase<BindingsClause>
+ implements IBindingProducerNode {
/**
*
@@ -178,7 +179,7 @@
sb.append(s);
- sb.append("BINDINGS");
+ sb.append("BindingsClause");
for(IVariable<?> var : declaredVars) {
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionRegistry.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionRegistry.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionRegistry.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -52,6 +52,7 @@
import com.bigdata.rdf.internal.constraints.MathBOp;
import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
import com.bigdata.rdf.internal.constraints.NotBOp;
+import com.bigdata.rdf.internal.constraints.NowBOp;
import com.bigdata.rdf.internal.constraints.NumericBOp;
import com.bigdata.rdf.internal.constraints.NumericBOp.NumericOp;
import com.bigdata.rdf.internal.constraints.OrBOp;
@@ -71,6 +72,7 @@
import com.bigdata.rdf.internal.constraints.StrstartsBOp;
import com.bigdata.rdf.internal.constraints.SubstrBOp;
import com.bigdata.rdf.internal.constraints.TrueBOp;
+import com.bigdata.rdf.internal.constraints.UUIDBOp;
import com.bigdata.rdf.internal.constraints.UcaseBOp;
import com.bigdata.rdf.internal.constraints.XsdStrBOp;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpUtility;
@@ -137,14 +139,15 @@
public static final URI IF = new URIImpl(SPARQL_FUNCTIONS+"if");
- public static final URI NOW = new URIImpl(SPARQL_FUNCTIONS+"now");
- public static final URI YEAR = new URIImpl(SPARQL_FUNCTIONS+"year-from-dateTime");
- public static final URI MONTH = new URIImpl(SPARQL_FUNCTIONS+"month-from-dateTime");
- public static final URI DAY = new URIImpl(SPARQL_FUNCTIONS+"day-from-dateTime");
- public static final URI HOURS = new URIImpl(SPARQL_FUNCTIONS+"hours-from-dateTime");
- public static final URI MINUTES = new URIImpl(SPARQL_FUNCTIONS+"minutes-from-dateTime");
- public static final URI SECONDS = new URIImpl(SPARQL_FUNCTIONS+"seconds-from-dateTime");
- public static final URI TIMEZONE = new URIImpl(SPARQL_FUNCTIONS+"tz");
+ public static final URI NOW = new URIImpl(XPATH_FUNCTIONS+"now");
+ public static final URI YEAR = new URIImpl(XPATH_FUNCTIONS+"year-from-dateTime");
+ public static final URI MONTH = new URIImpl(XPATH_FUNCTIONS+"month-from-dateTime");
+ public static final URI DAY = new URIImpl(XPATH_FUNCTIONS+"day-from-dateTime");
+ public static final URI HOURS = new URIImpl(XPATH_FUNCTIONS+"hours-from-dateTime");
+ public static final URI MINUTES = new URIImpl(XPATH_FUNCTIONS+"minutes-from-dateTime");
+ public static final URI SECONDS = new URIImpl(XPATH_FUNCTIONS+"seconds-from-dateTime");
+ public static final URI TZ = new URIImpl(XPATH_FUNCTIONS+"tz");
+ public static final URI TIMEZONE = new URIImpl(XPATH_FUNCTIONS+"timezone-from-dateTime");
public static final URI MD5 = new URIImpl(SPARQL_FUNCTIONS+"md5");
public static final URI SHA1 = new URIImpl(SPARQL_FUNCTIONS+"sha1");
@@ -153,6 +156,9 @@
public static final URI SHA384 = new URIImpl(SPARQL_FUNCTIONS+"sha384");
public static final URI SHA512 = new URIImpl(SPARQL_FUNCTIONS+"sha512");
+ public static final URI UUID = new URIImpl(SPARQL_FUNCTIONS+"uuid");
+ public static final URI STRUUID = new URIImpl(SPARQL_FUNCTIONS+"struuid");
+
public static final URI STR_DT = new URIImpl(SPARQL_FUNCTIONS+"strdt");
public static final URI STR_LANG = new URIImpl(SPARQL_FUNCTIONS+"strlang");
public static final URI LCASE = FN.LOWER_CASE;//new URIImpl(XPATH_FUNCTIONS+"lower-case");
@@ -209,7 +215,7 @@
public static final URI SUBTRACT = new URIImpl(XPATH_FUNCTIONS+"numeric-subtract");
public static final URI MULTIPLY = new URIImpl(XPATH_FUNCTIONS+"numeric-multiply");
public static final URI DIVIDE = new URIImpl(XPATH_FUNCTIONS+"numeric-divide");
-
+
public static final URI ABS = FN.NUMERIC_ABS;//new URIImpl(XPATH_FUNCTIONS+"numeric-abs");
public static final URI ROUND = FN.NUMERIC_ROUND;//new URIImpl(SPARQL_FUNCTIONS+"numeric-round");
public static final URI CEIL = FN.NUMERIC_CEIL;//new URIImpl(SPARQL_FUNCTIONS+"numeric-ceil");
@@ -709,8 +715,11 @@
// final IValueExpression<? extends IV> var = args[0].getValueExpression();
final IValueExpression ve = AST2BOpUtility.toVE(globals, args[0]);
+
+ final String baseURI = (String)
+ scalarValues.get(IriBOp.Annotations.BASE_URI);
- return new IriBOp(ve, globals);
+ return new IriBOp(ve, baseURI, globals);
}
});
@@ -917,6 +926,44 @@
add(TIMEZONE,new DateFactory(DateOp.TIMEZONE));
+ add(TZ,new DateFactory(DateOp.TZ));
+
+ add(NOW,new Factory() {
+ public IValueExpression<? extends IV> create(final GlobalAnnotations globals,
+ Map<String, Object> scalarValues, final ValueExpressionNode... args) {
+
+ if (args != null && args.length > 0)
+ throw new IllegalArgumentException("no args for NOW()");
+
+ return new NowBOp(globals);
+
+ }
+ });
+
+ add(UUID,new Factory() {
+ public IValueExpression<? extends IV> create(final GlobalAnnotations globals,
+ Map<String, Object> scalarValues, final ValueExpressionNode... args) {
+
+ if (args != null && args.length > 0)
+ throw new IllegalArgumentException("no args for UUID()");
+
+ return new UUIDBOp(globals, false);
+
+ }
+ });
+
+ add(STRUUID,new Factory() {
+ public IValueExpression<? extends IV> create(final GlobalAnnotations globals,
+ Map<String, Object> scalarValues, final ValueExpressionNode... args) {
+
+ if (args != null && args.length > 0)
+ throw new IllegalArgumentException("no args for STRUUID()");
+
+ return new UUIDBOp(globals, true);
+
+ }
+ });
+
add(MD5,new DigestFactory(DigestOp.MD5));
add(SHA1,new DigestFactory(DigestOp.SHA1));
@@ -1314,7 +1361,7 @@
final IValueExpression<? extends IV> left =
AST2BOpUtility.toVE(globals, args[0]);
- return new DateBOp(left, op);
+ return new DateBOp(left, op, globals);
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/GroupMemberValueExpressionNodeBase.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/GroupMemberValueExpressionNodeBase.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/GroupMemberValueExpressionNodeBase.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -74,7 +74,7 @@
}
- abstract IValueExpressionNode getValueExpressionNode();
+ abstract public IValueExpressionNode getValueExpressionNode();
abstract IValueExpression<? extends IV> getValueExpression();
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryBase.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryBase.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryBase.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -28,6 +28,7 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IVariable;
+import com.bigdata.rdf.sparql.ast.QueryRoot.Annotations;
/**
* Contains the projection clause, where clause, and solution modified clauses.
@@ -106,6 +107,11 @@
long DEFAULT_TIMEOUT = Long.MAX_VALUE;
+ /**
+ * The BINDINGS clause (optional).
+ */
+ String BINDINGS_CLAUSE = "bindingsClause";
+
}
/**
@@ -439,6 +445,26 @@
setProperty(Annotations.TIMEOUT, timeout);
}
+ /**
+ * Set the BINDINGS.
+ *
+ * @param bindings
+ */
+ public void setBindingsClause(final BindingsClause bindings) {
+
+ setProperty(Annotations.BINDINGS_CLAUSE, bindings);
+
+ }
+
+ /**
+ * Return the BINDINGS.
+ */
+ public BindingsClause getBindingsClause() {
+
+ return (BindingsClause) getProperty(Annotations.BINDINGS_CLAUSE);
+
+ }
+
public String toString(final int indent) {
final String s = indent(indent);
@@ -453,6 +479,7 @@
final HavingNode having = getHaving();
final OrderByNode orderBy = getOrderBy();
final SliceNode slice = getSlice();
+ final BindingsClause bindings = getBindingsClause();
if (getQueryType() != null) {
@@ -522,7 +549,13 @@
sb.append("=");
sb.append(getQueryHints().toString());
}
+
+ if (bindings != null) {
+ sb.append(bindings.toString(indent + 1));
+
+ }
+
return sb.toString();
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryRoot.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryRoot.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryRoot.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -56,11 +56,6 @@
*/
String NAMED_SUBQUERIES = "namedSubqueries";
- /**
- * The BINDINGS clause (optional).
- */
- String BINDINGS_CLAUSE = "bindingsClause";
-
}
/**
@@ -202,26 +197,6 @@
}
- /**
- * Set the BINDINGS.
- *
- * @param bindings
- */
- public void setBindingsClause(final BindingsClause bindings) {
-
- setProperty(Annotations.BINDINGS_CLAUSE, bindings);
-
- }
-
- /**
- * Return the BINDINGS.
- */
- public BindingsClause getBindingsClause() {
-
- return (BindingsClause) getProperty(Annotations.BINDINGS_CLAUSE);
-
- }
-
@Override
public String toString(final int indent) {
@@ -239,7 +214,7 @@
final DatasetNode dataset = getDataset();
- final BindingsClause bindings = getBindingsClause();
+// final BindingsClause bindings = getBindingsClause();
final NamedSubqueriesNode namedSubqueries = getNamedSubqueries();
@@ -327,12 +302,12 @@
sb.append(super.toString(indent));
- if (bindings != null) {
+// if (bindings != null) {
+//
+// sb.append(bindings.toString(indent + 1));
+//
+// }
- sb.append(bindings.toString(indent + 1));
-
- }
-
return sb.toString();
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -1066,6 +1066,12 @@
// NOP
+ } else if(node instanceof BindingsClause) {
+
+ final BindingsClause bc = (BindingsClause) node;
+
+ vars.addAll(bc.getDeclaredVariables());
+
} else {
throw new AssertionError(node.toString());
@@ -1165,6 +1171,10 @@
// NOP
+ } else if(child instanceof BindingsClause) {
+
+ // NOP
+
} else {
throw new AssertionError(child.toString());
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -267,7 +267,7 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/558
*/
op = (Update) new ASTBatchResolveTermsOptimizer().optimize(context,
- op/* queryNode */, null/* bindingSets */);
+ op/* queryNode */, context.getBindings()/* bindingSets */);
}
@@ -486,6 +486,19 @@
new LinkedHashSet<IVariable<?>>()/* vars */,
true/* recursive */);
+ for (IBindingSet bs : context.getBindings()) {
+
+ @SuppressWarnings("rawtypes")
+ final Iterator<IVariable> it = bs.vars();
+
+ while (it.hasNext()) {
+
+ projectedVars.add(it.next());
+
+ }
+
+ }
+
final ProjectionNode projection = new ProjectionNode();
for (IVariable<?> var : projectedVars) {
@@ -493,7 +506,7 @@
projection.addProjectionVar(new VarNode(var.getName()));
}
-
+
queryRoot.setProjection(projection);
}
@@ -603,7 +616,7 @@
final MutableTupleQueryResult result = new MutableTupleQueryResult(
ASTEvalHelper.evaluateTupleQuery(
context.conn.getTripleStore(), astContainer,
- null/* bindingSets */));
+ context.getQueryBindingSet()/* bindingSets */));
boolean nativeDistinct = astContainer.getOptimizedAST().getProperty(ConstructNode.Annotations.NATIVE_DISTINCT,
ConstructNode.Annotations.DEFAULT_NATIVE_DISTINCT);
@@ -716,7 +729,7 @@
.evaluateTupleQuery2(
context.conn.getTripleStore(),
astContainer,
- null/* bindingSets */, false/* materialize */);
+ context.getQueryBindingSet()/* bindingSets */, false/* materialize */);
try {
@@ -861,6 +874,7 @@
} else {
+
/*
* DELETE/INSERT.
*
@@ -916,7 +930,7 @@
*/
final ICloseableIterator<IBindingSet[]> result = ASTEvalHelper
.evaluateTupleQuery2(context.conn.getTripleStore(),
- astContainer, null/* bindingSets */, false/* materialize */);
+ astContainer, context.getQueryBindingSet()/* bindingSets */, false/* materialize */);
try {
@@ -947,7 +961,7 @@
// Set the CONSTRUCT template (quads patterns).
queryRoot.setConstruct(template);
-
+
/*
* Run as a CONSTRUCT query
*
@@ -963,8 +977,8 @@
*/
final GraphQueryResult result = ASTEvalHelper
.evaluateGraphQuery(context.conn.getTripleStore(),
- astContainer, null/* bindingSets */);
-
+ astContainer, context.getQueryBindingSet()/* bindingSets */);
+
try {
while (result.hasNext()) {
@@ -1149,7 +1163,7 @@
if (!silent) {
- assertGraphNotEmpty(context, sourceGraph);
+// assertGraphNotEmpty(context, sourceGraph);
}
@@ -1357,7 +1371,7 @@
/*
* Execute the update.
*/
- executeUpdate(left, null/* bindingSets */, context);
+ executeUpdate(left, context.getBindings()/* bindingSets */, context);
// Return null since pipeline was evaluated.
return null;
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -29,8 +29,10 @@
import java.util.concurrent.atomic.AtomicLong;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
import org.openrdf.sail.SailException;
+import com.bigdata.bop.IBindingSet;
import com.bigdata.journal.ITx;
import com.bigdata.rdf.internal.IVCache;
import com.bigdata.rdf.model.BigdataURI;
@@ -55,6 +57,10 @@
public final BigdataSailRepositoryConnection conn;
private boolean includeInferred;
+
+ private QueryBindingSet qbs;
+
+ private IBindingSet[] bindingSets;
/**
* The timestamp associated with the commit point for the update and
@@ -145,6 +151,30 @@
}
+ public IBindingSet[] getBindings() {
+
+ return bindingSets;
+
+ }
+
+ public void setBindings(final IBindingSet[] bindingSets) {
+
+ this.bindingSets = bindingSets;
+
+ }
+
+ public QueryBindingSet getQueryBindingSet() {
+
+ return qbs;
+
+ }
+
+ public void setQueryBindingSet(final QueryBindingSet qbs) {
+
+ this.qbs = qbs;
+
+ }
+
private BigdataURI nullGraph = null;
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -92,6 +92,7 @@
import com.bigdata.rdf.internal.constraints.ProjectedConstraint;
import com.bigdata.rdf.internal.constraints.SPARQLConstraint;
import com.bigdata.rdf.internal.constraints.TryBeforeMaterializationConstraint;
+import com.bigdata.rdf.internal.constraints.UUIDBOp;
import com.bigdata.rdf.internal.impl.literal.XSDBooleanIV;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataURI;
@@ -99,6 +100,7 @@
import com.bigdata.rdf.sparql.ast.ASTUtil;
import com.bigdata.rdf.sparql.ast.ArbitraryLengthPathNode;
import com.bigdata.rdf.sparql.ast.AssignmentNode;
+import com.bigdata.rdf.sparql.ast.BindingsClause;
import com.bigdata.rdf.sparql.ast.ComputedMaterializationRequirement;
import com.bigdata.rdf.sparql.ast.ConstantNode;
import com.bigdata.rdf.sparql.ast.DatasetNode;
@@ -2687,6 +2689,11 @@
left = addAssignment(left, (AssignmentNode) child, doneSet,
joinGroup.getQueryHints(), ctx, false/* projection */);
continue;
+ } else if (child instanceof BindingsClause) {
+// // LET / BIND
+// left = addAssignment(left, (AssignmentNode) child, doneSet,
+// joinGroup.getQueryHints(), ctx, false/* projection */);
+ continue;
} else {
throw new UnsupportedOperationException("child: " + child);
}
@@ -4575,6 +4582,16 @@
return ve;
}
+
+ if (op instanceof UUIDBOp) {
+
+ /*
+ * We cannot pre-generate these, they need to be unique
+ * for each call.
+ */
+ return ve;
+
+ }
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2014-07-16 15:50:37 UTC (rev 8557)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2014-07-16 16:01:00 UTC (rev 8558)
@@ -29,6 +29,7 @@
import info.aduna.iteration.CloseableIteration;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -80,9 +81,14 @@
import com.bigdata.rdf.sparql....
[truncated message content] |
|
From: <tho...@us...> - 2014-07-16 15:50:48
|
Revision: 8557
http://sourceforge.net/p/bigdata/code/8557
Author: thompsonbry
Date: 2014-07-16 15:50:37 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
I have added support for hierarchical locking of the resources in a namespace to the AbstractTask class. There is a new test suite for hierarchical locking support.
We need to improve the GIST capabilities of AbstractTask before we can use it with RDF KBs. Right now, it only supports the B+Tree class and makes assumptions about ILocalBTreeView as the index type. The named solution set mechanisms use the HTree and Stream objects as well. Similar GIST support was introduced to the Journal about a year ago.
The journal, service, and NSS test suites are green.
See #585 GIST (Generalized Indices)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/service/TestAll.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-07-16 15:37:58 UTC (rev 8556)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-07-16 15:50:37 UTC (rev 8557)
@@ -387,10 +387,12 @@
for(String s : resource) {
final Name2Addr.Entry tmp = name2Addr.getEntry(s);
+
+ if (tmp != null) {
- if(tmp != null) {
-
/*
+ * Exact match on a named index.
+ *
* Add a read-only copy of the entry with additional state
* for tracking registration and dropping of named indices.
*
@@ -400,6 +402,37 @@
n2a.put(s, new Entry(tmp));
+ } else {
+
+ /**
+ * Add a read-only copy of the Name2Addr entry for all
+ * entries spanned by that namespace. This provides the
+ * additional state for tracking registration and dropping
+ * of named indices and also supports hierarchical locking
+ * pattersn.
+ *
+ * Note: We do NOT fetch the indices here, just copy their
+ * last checkpoint metadata from Name2Addr.
+ *
+ * @see <a
+ * href="http://trac.bigdata.com/ticket/566"
+ * > Concurrent unisolated operations against multiple
+ * KBs </a>
+ */
+
+ final Iterator<String> itr = Name2Addr.indexNameScan(s,
+ name2Addr);
+
+ while (itr.hasNext()) {
+
+ final String t = itr.next();
+
+ final Name2Addr.Entry tmp2 = name2Addr.getEntry(t);
+
+ n2a.put(t, new Entry(tmp2));
+
+ }
+
}
}
@@ -517,7 +550,7 @@
* delegated to this method. First, the task can use this method directly.
* Second, the task can use {@link #getJournal()} and then use
* {@link IJournal#getIndex(String)} on that journal, which is simply
- * delegated to this method. See {@link IsolatedActionJournal}.
+ * delegated to this method. See {@link IsolatedActionJournal}.
*
* @param name
* The name of the index.
@@ -534,8 +567,12 @@
*
* @return The index.
*
- * @todo modify to return <code>null</code> if the index is not
- * registered?
+ * @todo modify to return <code>null</code> if the index is not registered?
+ *
+ * FIXME GIST. This will throw a ClassCastException if the returned
+ * index is an ILocalBTreeView.
+ *
+ * @see http://trac.bigdata.com/ticket/585 (GIST)
*/
@Override
synchronized final public ILocalBTreeView getIndex(final String name) {
@@ -955,7 +992,7 @@
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
*
- * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/675" >
+ * @see <a href="http://trac.bigdata.com/ticket/675" >
* Flush indices in parallel during checkpoint to reduce IO latency</a>
*/
private class CheckpointIndexTask implements Callable<Void> {
@@ -1003,7 +1040,7 @@
*
* @return The elapsed time in nanoseconds for this operation.
*
- * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/675"
+ * @see <a href="http://trac.bigdata.com/ticket/675"
* >Flush indices in parallel during checkpoint to reduce IO
* latency</a>
*/
@@ -1315,13 +1352,18 @@
* The transaction identifier -or- {@link ITx#UNISOLATED} IFF the
* operation is NOT isolated by a transaction -or-
* <code> - timestamp </code> to read from the most recent commit
- * point not later than the absolute value of <i>timestamp</i>
- * (a historical read).
+ * point not later than the absolute value of <i>timestamp</i> (a
+ * historical read).
* @param resource
* The resource on which the task will operate. E.g., the names
* of the index. When the task is an unisolated write task an
* exclusive lock will be requested on the named resource and the
* task will NOT run until it has obtained that lock.
+ * <p>
+ * The name may identify either a namespace or a concrete index
+ * object. If a concrete index object is discovered, only that
+ * index is isolated. Otherwise all indices having the same
+ * prefix as the namespace are isolated.
*/
protected AbstractTask(final IConcurrencyManager concurrencyManager,
final long timestamp, final String resource) {
@@ -1346,6 +1388,11 @@
* task an exclusive lock will be requested on each named
* resource and the task will NOT run until it has obtained those
* lock(s).
+ * <p>
+ * The name may identify either a namespace or a concrete index
+ * object. If a concrete index object is discovered, only that
+ * index is isolated. Otherwise all indices having the same
+ * prefix as the namespace are isolated.
*/
protected AbstractTask(final IConcurrencyManager concurrencyManager,
final long timestamp, final String[] resource) {
@@ -1453,7 +1500,7 @@
if (transactionManager.getTx(timestamp) == null) {
- /*
+ /**
* Start tx on this data service.
*
* FIXME This should be passing the [readsOnCommitTime] into
@@ -1464,11 +1511,11 @@
* submitted primarily for the clustered database
* deployment.
*
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/266
+ * @see http://trac.bigdata.com/ticket/266
* (refactor native long tx id to thin object)
*
* @see <a
- * href="http://sourceforge.net/apps/trac/bigdata/ticket/546"
+ * href="http://trac.bigdata.com/ticket/546"
* > Add cache for access to historical index views on the
* Journal by name and commitTime. </a>
*/
@@ -1588,41 +1635,40 @@
}
/**
- * FIXME GROUP_COMMIT: Supporting this requires us to support
- * efficient scans of the indices in Name2Addr having the prefix
- * values declared by [resources] since getIndex(name) will fail if
- * the Name2Addr entry has not been buffered within the [n2a] cache.
+ * Look for a prefix that spans one or more resources.
*
- * @see <a
- * href="http://sourceforge.net/apps/trac/bigdata/ticket/753" >
- * HA doLocalAbort() should interrupt NSS requests and
+ * Note: Supporting this requires us to support efficient scans of
+ * the indices in Name2Addr since getIndex(name) will fail if the
+ * Name2Addr entry has not been buffered within the [n2a] cache.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/753" > HA
+ * doLocalAbort() should interrupt NSS requests and
* AbstractTasks </a>
- * @see <a
- * href="- http://sourceforge.net/apps/trac/bigdata/ticket/566"
- * > Concurrent unisolated operations against multiple KBs </a>
+ * @see <a href="http://trac.bigdata.com/ticket/566" > Concurrent
+ * unisolated operations against multiple KBs </a>
*/
-// if (theRequestedResource.startsWith(theDeclaredResource)) {
-//
-// // Possible prefix match.
-//
-// if (theRequestedResource.charAt(theDeclaredResource.length()) == '.') {
-//
-// /*
-// * Prefix match.
-// *
-// * E.g., name:="kb.spo.osp" and the task declared the
-// * resource "kb". In this case, "kb" is a PREFIX of the
-// * declared resource and the next character is the separator
-// * character for the resource names (this last point is
-// * important to avoid unintended contention between
-// * namespaces such as "kb" and "kb1").
-// */
-// return true;
-//
-// }
-//
-// }
+ if (theRequestedResource.startsWith(theDeclaredResource)) {
+
+ // Possible prefix match.
+
+ if (theRequestedResource.charAt(theDeclaredResource.length()) == '.') {
+ /*
+ * Prefix match.
+ *
+ * E.g., name:="kb.spo.osp" and the task declared the
+ * resource "kb". In this case, "kb" is a PREFIX of the
+ * declared resource and the next character is the separator
+ * character for the resource names (this last point is
+ * important to avoid unintended contention between
+ * namespaces such as "kb" and "kb1").
+ */
+ return true;
+
+ }
+
+ }
+
}
return false;
@@ -2488,7 +2534,7 @@
* FIXME GIST : Support registration of index types other than BTree
* (HTree, Stream, etc).
*
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/585 (GIST)
+ * @see http://trac.bigdata.com/ticket/585 (GIST)
*/
throw new UnsupportedOperationException();
@@ -2522,14 +2568,14 @@
* Note: access to an unisolated index is governed by the AbstractTask.
*/
@Override
- public ICheckpointProtocol getUnisolatedIndex(String name) {
+ public ICheckpointProtocol getUnisolatedIndex(final String name) {
try {
/*
* FIXME GIST. This will throw a ClassCastException if the
* returned index is an ILocalBTreeView.
*
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/585 (GIST)
+ * @see http://trac.bigdata.com/ticket/585 (GIST)
*/
return (ICheckpointProtocol) AbstractTask.this.getIndex(name);
@@ -2599,7 +2645,7 @@
* in a ClassCastException.
*
* @see <a
- * href="https://sourceforge.net/apps/trac/bigdata/ticket/585"
+ * href="http://trac.bigdata.com/ticket/585"
* > GIST </a>
*/
return (ICheckpointProtocol) resourceManager.getIndex(name, commitTime);
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestHierarchicalLockingTasks.java 2014-07-16 15:50:37 UTC (rev 8557)
@@ -0,0 +1,378 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Oct 15, 2007
+ */
+
+package com.bigdata.journal;
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import com.bigdata.btree.IIndex;
+import com.bigdata.btree.ILocalBTreeView;
+import com.bigdata.btree.IndexMetadata;
+
+/**
+ * Test suite for hierarchical locking of indices based on namespace prefixes.
+ * This test suite was introduced as part of the support for group commit for
+ * the RDF database layer in the non-scale-out modes. In order to be able to run
+ * RDF operations using job-based concurrency, we need to predeclare the
+ * namespace (rather than the individual indices) and then isolate all indices
+ * spanned by that namespace.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ *
+ * @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
+ * Concurrent unisolated operations against multiple KBs </a>
+ *
+ * FIXME GROUP COMMIT (hierarchical locking): We need to test each
+ * different kind of index access here (unisolated, isolated by a
+ * read/write tx, and isolated by a read-only tx, and read-historical (no
+ * isolation)).
+ */
+public class TestHierarchicalLockingTasks extends ProxyTestCase<Journal> {
+
+ /**
+ *
+ */
+ public TestHierarchicalLockingTasks() {
+ super();
+ }
+
+ /**
+ * @param name
+ */
+ public TestHierarchicalLockingTasks(final String name) {
+ super(name);
+ }
+
+ /**
+ * Test creates several named indices some of which have a shared namespace
+ * prefix and verifies that a lock declared for the namespace prefix permits
+ * that task to access any index in that namespace. The test also verifies
+ * that an index that does not share the namespace prefix may not be
+ * accessed by the task.
+ *
+ * @throws InterruptedException
+ * @throws ExecutionException
+ */
+ public void test_hierarchicalLocking_001() throws InterruptedException,
+ ExecutionException {
+
+ // namespace for declared locks.
+ final String[] namespace = new String[] { "foo" };
+
+ // indices that we create, write on, and verify.
+ final String[] allowed_indices = new String[] { "foo.bar", "foo.baz" };
+
+ // indices that are outside of the declared namespace. we verify that we
+ // can not access these indices.
+ final String[] disallowed_indices = new String[] { "goo", "goo.bar" };
+
+ final Journal journal = getStore();
+
+ try {
+
+ /*
+ * Create indices that we should not be able to see when holding
+ * just the lock for the namespace.
+ */
+ for (int i = 0; i < disallowed_indices.length; i++) {
+
+ final String name = disallowed_indices[i];
+
+ journal.submit(new RegisterIndexTask(journal
+ .getConcurrencyManager(), name, new IndexMetadata(name,
+ UUID.randomUUID()))).get();
+
+ }
+
+ /*
+ * Create indices that we should be able to see when holding the
+ * lock for the namespace.
+ */
+ for (int i = 0; i < allowed_indices.length; i++) {
+
+ final String name = allowed_indices[i];
+
+ journal.submit(new RegisterIndexTask(journal
+ .getConcurrencyManager(), name, new IndexMetadata(name,
+ UUID.randomUUID()))).get();
+
+ }
+
+ /*
+ * Submit task holding the lock for the namespace and verify that we
+ * can see the indices that are spanned by the namespace, but not
+ * those that are not spanned by the namespace.
+ */
+
+ final Future<Void> ft = journal
+ .submit(new AbstractTask<Void>(journal
+ .getConcurrencyManager(), ITx.UNISOLATED, namespace) {
+
+ @Override
+ protected Void doTask() throws Exception {
+
+ // Verify access to the indices in that namespace.
+ for (String name : allowed_indices) {
+
+ assertNotNull(name, getIndex(name));
+
+ }
+
+ /*
+ * Verify no access to the indices outside of that
+ * namespace.
+ */
+ for (String name : disallowed_indices) {
+
+ try {
+ // Attempt to access index.
+ getIndex(name);
+ fail("Expecting: "
+ + IllegalStateException.class
+ + " for " + name);
+ } catch (IllegalStateException ex) {
+ // log and ignore expected exception.
+ if (log.isInfoEnabled())
+ log.info("Ignoring expected exception");
+ }
+
+ }
+
+ // Done.
+ return null;
+ }
+ });
+
+ try {
+ // Await outcome.
+ ft.get();
+ } finally {
+ ft.cancel(true/* mayInterruptIfRunning */);
+ }
+
+ } finally {
+
+ journal.destroy();
+
+ }
+
+ }
+
+ /**
+ * Unit test for hierarchical locking verifies that we can declared a
+ * namespace for a task which then creates multiple indices spanned by that
+ * namespace.
+ *
+ * @throws InterruptedException
+ * @throws ExecutionException
+ */
+ public void test_hierarchicalLocking_create_destroy()
+ throws InterruptedException, ExecutionException {
+
+ // namespace for declared locks.
+ final String[] namespace = new String[] { "foo" };
+
+ // indices that we create, write on, and verify.
+ final String[] allowed_indices = new String[] { "foo.bar", "foo.baz" };
+
+ final Journal journal = getStore();
+
+ try {
+
+ final UUID[] uuids = journal.submit(
+ new AbstractTask<UUID[]>(journal.getConcurrencyManager(),
+ ITx.UNISOLATED, namespace) {
+
+ @Override
+ protected UUID[] doTask() throws Exception {
+
+ final UUID[] indexUUIDs = new UUID[allowed_indices.length];
+
+ /*
+ * Create indices that we should be able to see when
+ * holding the lock for the namespace.
+ */
+ for (int i = 0; i < allowed_indices.length; i++) {
+
+ final String name = allowed_indices[i];
+
+ IIndex ndx = getJournal().getIndex(name);
+
+ if (ndx != null) {
+
+ final UUID indexUUID = ndx.getIndexMetadata().getIndexUUID();
+
+ if (log.isInfoEnabled())
+ log.info("Index exists: name=" + name + ", indexUUID=" + indexUUID);
+
+ indexUUIDs[i] = indexUUID;
+
+ }
+
+ // register the index.
+ ndx = getJournal().registerIndex(
+ name,
+ new IndexMetadata(name, UUID
+ .randomUUID()));
+
+ final UUID indexUUID = ndx.getIndexMetadata().getIndexUUID();
+
+ if (log.isInfoEnabled())
+ log.info("Registered index: name=" + name + ", class="
+ + ndx.getClass() + ", indexUUID=" + indexUUID);
+
+ indexUUIDs[i] = indexUUID;
+
+ }
+
+ // Done
+ return indexUUIDs;
+ }
+
+ }).get();
+
+ // should be non-null.
+ assertNotNull(uuids);
+
+ // Should be non-null.
+ for (UUID uuid : uuids) {
+
+ assertNotNull(uuid);
+
+ }
+
+ /*
+ * Verify access to the newly created indices.
+ */
+ journal.submit(
+ new AbstractTask<Void>(journal.getConcurrencyManager(),
+ ITx.UNISOLATED, namespace) {
+
+ @Override
+ protected Void doTask() throws Exception {
+
+ /*
+ * Verify access to the newly created indices.
+ */
+ for (int i = 0; i < allowed_indices.length; i++) {
+
+ final String name = allowed_indices[i];
+
+ final ILocalBTreeView ndx = getIndex(name);
+
+ assertNotNull(ndx);
+
+ assertEquals(0L, ndx.rangeCount());
+
+ // add a key.
+ ndx.insert(new byte[] {},
+ new byte[] { (byte) i });
+
+ }
+
+ // Done
+ return null;
+ }
+
+ }).get();
+
+ /*
+ * Destroy the newly created indices.
+ */
+ journal.submit(
+ new AbstractTask<Void>(journal.getConcurrencyManager(),
+ ITx.UNISOLATED, namespace) {
+
+ @Override
+ protected Void doTask() throws Exception {
+
+ /*
+ * Create indices that we should be able to see when
+ * holding the lock for the namespace.
+ */
+ for (int i = 0; i < allowed_indices.length; i++) {
+
+ final String name = allowed_indices[i];
+
+ getJournal().dropIndex(name);
+
+ }
+
+ // Done
+ return null;
+ }
+
+ }).get();
+
+ /*
+ * Verify that the indices can no longer be accessed.
+ */
+ journal.submit(
+ new AbstractTask<Void>(journal.getConcurrencyManager(),
+ ITx.UNISOLATED, namespace) {
+
+ @Override
+ protected Void doTask() throws Exception {
+
+ /*
+ * Verify no access to the dropped indices.
+ */
+ for (int i = 0; i < allowed_indices.length; i++) {
+
+ final String name = allowed_indices[i];
+
+ try {
+ // Attempt to access index.
+ getIndex(name);
+ fail("Expecting: "
+ + IllegalStateException.class
+ + " for " + name);
+ } catch (IllegalStateException ex) {
+ // log and ignore expected exception.
+ if (log.isInfoEnabled())
+ log.info("Ignoring expected exception");
+ }
+
+ }
+
+ // Done
+ return null;
+ }
+
+ }).get();
+
+ } finally {
+
+ journal.destroy();
+
+ }
+
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2014-07-16 15:37:58 UTC (rev 8556)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2014-07-16 15:50:37 UTC (rev 8557)
@@ -125,15 +125,25 @@
// suite.addTestSuite(TestAddDropIndexTask.class);
// test writing on one or more unisolated indices and verify read back after the commit.
suite.addTestSuite(TestUnisolatedWriteTasks.class);
+ // test suite for hierarchical locking (namespace prefixes).
+ suite.addTestSuite(TestHierarchicalLockingTasks.class);
+ // test suite for GIST operations using group commit.
+// suite.addTestSuite(TestGISTTasks.class);
// stress test of throughput when lock contention serializes unisolated writers.
suite.addTestSuite(StressTestLockContention.class);
// stress test of group commit.
suite.addTestSuite(StressTestGroupCommit.class);
// stress tests of writes on unisolated named indices using ConcurrencyManager.
suite.addTestSuite(StressTestConcurrentUnisolatedIndices.class);
- // stress tests of writes on unisolated named indices using UnisolatedReadWriteIndex.
- suite.addTestSuite(StressTestConcurrentUnisolatedIndices.class);
/*
+ * Stress tests of writes on unisolated named indices using
+ * UnisolatedReadWriteIndex.
+ *
+ * FIXME This test appears to cause #343 (Stochastic assert in
+ * AbstractBTree#writeNodeOrLeaf() in CI)
+ */
+// suite.addTestSuite(StressTestUnisolatedReadWriteIndex.class);
+ /*
* Stress test of concurrent transactions.
*
* Note: transactions use unisolated operations on the live indices when
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/service/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/service/TestAll.java 2014-07-16 15:37:58 UTC (rev 8556)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/service/TestAll.java 2014-07-16 15:50:37 UTC (rev 8557)
@@ -28,8 +28,6 @@
package com.bigdata.service;
-
-
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
@@ -38,7 +36,6 @@
* Test suite for embedded services.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class TestAll extends TestCase {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-16 15:38:06
|
Revision: 8556
http://sourceforge.net/p/bigdata/code/8556
Author: thompsonbry
Date: 2014-07-16 15:37:58 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
javadoc (bad links to tickets)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java 2014-07-16 15:37:38 UTC (rev 8555)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java 2014-07-16 15:37:58 UTC (rev 8556)
@@ -55,8 +55,8 @@
* request and response.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
- * Concurrent unisolated operations against multiple KBs </a>
+ * @see <a href="http://trac.bigdata.com/ticket/566" > Concurrent unisolated
+ * operations against multiple KBs </a>
*/
abstract public class AbstractApiTask<T> implements IApiTask<T>, IReadOnly {
@@ -250,10 +250,10 @@
*
* @throws DatasetNotFoundException
*
- * @see <a href="http://sourceforge.net/apps/trac/bigdata/ticket/753" > HA
- * doLocalAbort() should interrupt NSS requests and AbstractTasks </a>
- * @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
- * Concurrent unisolated operations against multiple KBs </a>
+ * @see <a href="http://trac.bigdata.com/ticket/753" > HA doLocalAbort()
+ * should interrupt NSS requests and AbstractTasks </a>
+ * @see <a href="http://trac.bigdata.com/ticket/566" > Concurrent unisolated
+ * operations against multiple KBs </a>
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
static public <T> Future<T> submitApiTask(
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-16 15:37:43
|
Revision: 8555
http://sourceforge.net/p/bigdata/code/8555
Author: thompsonbry
Date: 2014-07-16 15:37:38 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
Code cleanup on Drop/Register index tasks.
See #566 (Group commit support for RDF)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/DropIndexTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/DropIndexTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/DropIndexTask.java 2014-07-16 14:56:31 UTC (rev 8554)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/DropIndexTask.java 2014-07-16 15:37:38 UTC (rev 8555)
@@ -33,11 +33,13 @@
* execute after the index has been dropped will NOT be able to see the index.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ *
+ * @see RegisterIndexTask
*/
-public class DropIndexTask extends AbstractTask {
+public class DropIndexTask extends AbstractTask<Boolean> {
- public DropIndexTask(IConcurrencyManager concurrencyManager, String name) {
+ public DropIndexTask(final IConcurrencyManager concurrencyManager,
+ final String name) {
super(concurrencyManager, ITx.UNISOLATED, name);
@@ -51,9 +53,10 @@
* therefore was dropped. <code>false</code> is returned iff the
* index did not exist at the time that this task was executed.
*/
- public Object doTask() throws Exception {
+ @Override
+ public Boolean doTask() throws Exception {
- String name = getOnlyResource();
+ final String name = getOnlyResource();
try {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java 2014-07-16 14:56:31 UTC (rev 8554)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java 2014-07-16 15:37:38 UTC (rev 8555)
@@ -46,7 +46,8 @@
* registered index.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ *
+ * @see DropIndexTask
*/
public class RegisterIndexTask extends AbstractTask<UUID> {
@@ -81,6 +82,7 @@
*
* @return The {@link UUID} of the named index.
*/
+ @Override
protected UUID doTask() throws Exception {
final String name = getOnlyResource();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-16 14:56:41
|
Revision: 8554
http://sourceforge.net/p/bigdata/code/8554
Author: thompsonbry
Date: 2014-07-16 14:56:31 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
Extracted an IReadOnly interface from IIndexProcedure.
Removed the sub-classes of AbstractRestApiTask that marked read-only versus read-write tasks and replaced them with the IReadOnly iterface.
Cleanup around the group commit code and test suites.
See #566 (Concurrent unisolated operations against multiple KBs on the same Journal)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentTx.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentUnisolatedIndices.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestLockContention.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/ApiTaskForIndexManager.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/AbstractRestApiTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/UpdateServlet.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IReadOnly.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -34,6 +34,7 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ISimpleBTree;
+import com.bigdata.journal.IReadOnly;
import com.bigdata.service.DataService;
import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.ClientIndexView;
@@ -94,23 +95,22 @@
* with the JINI codebase mechanism (<code>java.rmi.server.codebase</code>).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo add generic type for {@link #apply(IIndex)} 's return value (much like
* {@link Callable}).
*/
-public interface IIndexProcedure extends Serializable {
+public interface IIndexProcedure extends IReadOnly, Serializable {
- /**
- * Return <code>true</code> iff the procedure asserts that it will not
- * write on the index. When <code>true</code>, the procedure may be run
- * against a view of the index that is read-only or which allows concurrent
- * processes to read on the same index object. When <code>false</code> the
- * procedure will be run against a mutable view of the index (assuming that
- * the procedure is executed in a context that has access to a mutable index
- * view).
- */
- public boolean isReadOnly();
+// /**
+// * Return <code>true</code> iff the procedure asserts that it will not
+// * write on the index. When <code>true</code>, the procedure may be run
+// * against a view of the index that is read-only or which allows concurrent
+// * processes to read on the same index object. When <code>false</code> the
+// * procedure will be run against a mutable view of the index (assuming that
+// * the procedure is executed in a context that has access to a mutable index
+// * view).
+// */
+// public boolean isReadOnly();
/**
* Run the procedure.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -62,8 +62,6 @@
import com.bigdata.btree.ILocalBTreeView;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.view.FusedView;
-import com.bigdata.concurrent.LockManager;
-import com.bigdata.concurrent.LockManagerTask;
import com.bigdata.concurrent.NonBlockingLockManager;
import com.bigdata.counters.CounterSet;
import com.bigdata.mdi.IResourceMetadata;
@@ -116,7 +114,6 @@
* {@link ConcurrencyManager#submit(AbstractTask)} it.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo declare generic type for the return as <? extends Object> to be compatible
* with {@link ConcurrencyManager#submit(AbstractTask)}
@@ -166,12 +163,14 @@
* The object used to manage access to the resources from which views of the
* indices are created.
*/
+ @Override
public final IResourceManager getResourceManager() {
return resourceManager;
}
+ @Override
synchronized public final IJournal getJournal() {
if (journal == null) {
@@ -282,7 +281,7 @@
boolean registeredIndex = false;
boolean droppedIndex = false;
- Entry(Name2Addr.Entry entry) {
+ Entry(final Name2Addr.Entry entry) {
super(entry.name, entry.checkpointAddr, entry.commitTime);
@@ -425,6 +424,7 @@
final String name;
final ICheckpointProtocol btree;
+ @Override
public String toString() {
return "DirtyListener{name="+name+"}";
@@ -448,8 +448,9 @@
* Add <i>this</i> to the {@link AbstractTask#commitList}.
*
* @param btree
- * The {@link BTree} reporting that it is dirty.
+ * The index reporting that it is dirty.
*/
+ @Override
public void dirtyEvent(final ICheckpointProtocol btree) {
assert btree == this.btree;
@@ -966,7 +967,8 @@
this.l = l;
}
-
+
+ @Override
public Void call() throws Exception {
if(log.isInfoEnabled())
@@ -1240,8 +1242,10 @@
* fails. The boolean argument indicates whether or not the group commit
* succeeded. Throws exceptions are trapped and logged.
*/
- void afterTaskHook(boolean abort) {
+ void afterTaskHook(final boolean abort) {
+
((IsolatedActionJournal) getJournal()).completeTask();
+
}
/*
@@ -1260,6 +1264,7 @@
*/
protected TaskCounters taskCounters;
+ @Override
public TaskCounters getTaskCounters() {
return taskCounters;
@@ -1752,8 +1757,8 @@
* <dt>timestamp</dt>
* <dd>The {@link #timestamp} specified to the ctor.</dd>
* <dt>resources</dt>
- * <dd>The named resource(s) specified to the ctor IFF {@link #INFO} is
- * <code>true</code></dd>
+ * <dd>The named resource(s) specified to the ctor IFF logging @ INFO or
+ * above.</dd>
* </dl>
*/
protected void setupLoggingContext() {
@@ -1927,7 +1932,7 @@
* Delegate handles handshaking for writable transactions.
*/
- final Callable<T> delegate = new InnerReadWriteTxServiceCallable(
+ final Callable<T> delegate = new InnerReadWriteTxServiceCallable<T>(
this, tx);
return delegate.call();
@@ -1979,8 +1984,6 @@
* Call {@link #doTask()} for an unisolated write task.
*
* @throws Exception
- *
- * @todo update javadoc to reflect the change in how the locks are acquired.
*/
private T doUnisolatedReadWriteTask() throws Exception {
@@ -2074,7 +2077,7 @@
* access to the same resources since the locks were released above.
*/
- writeService.afterTask(this, null);
+ writeService.afterTask(this/* task */, null/* cause */);
return ret;
@@ -2087,7 +2090,7 @@
if (log.isInfoEnabled())
log.info("Task failed: class=" + this + " : " + t2);
- writeService.afterTask(this, t2);
+ writeService.afterTask(this/* task */, t2/* cause */);
}
@@ -2184,11 +2187,11 @@
* Inner class used to wrap up the call to {@link AbstractTask#doTask()} for
* read-write transactions.
*/
- static protected class InnerReadWriteTxServiceCallable extends DelegateTask {
+ static protected class InnerReadWriteTxServiceCallable<T> extends DelegateTask<T> {
- final Tx tx;
+ private final Tx tx;
- InnerReadWriteTxServiceCallable(AbstractTask delegate, Tx tx) {
+ InnerReadWriteTxServiceCallable(final AbstractTask<T> delegate, final Tx tx) {
super( delegate );
@@ -2202,7 +2205,8 @@
/**
* Wraps up the execution of {@link AbstractTask#doTask()}.
*/
- public Object call() throws Exception {
+ @Override
+ public T call() throws Exception {
// invoke on the outer class.
@@ -2234,9 +2238,8 @@
}
/**
- * An instance of this class is used as the delegate for a
- * {@link LockManagerTask} in order to coordinate the acquisition of locks
- * with the {@link LockManager} before the task can execute and to release
+ * An instance of this class is used as the delegate to coordinate the acquisition of locks
+ * with the {@link NonBlockingLockManager} before the task can execute and to release
* locks after the task has completed (whether it succeeds or fails).
* <p>
* Note: This inner class delegates the execution of the task to
@@ -2249,8 +2252,6 @@
* prevent tasks from progressing. If there is strong lock contention then
* writers will be more or less serialized.
*
- * @todo javadoc update to reflect the {@link NonBlockingLockManager}
- *
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
static protected class InnerWriteServiceCallable<T> extends DelegateTask<T> {
@@ -2264,6 +2265,7 @@
/**
* Note: Locks on the named indices are ONLY held during this call.
*/
+ @Override
public T call() throws Exception {
// The write service on which this task is running.
@@ -2356,7 +2358,7 @@
super();
}
- public ResubmitException(String msg) {
+ public ResubmitException(final String msg) {
super(msg);
@@ -2389,6 +2391,7 @@
@SuppressWarnings("rawtypes")
private final IResourceLocator resourceLocator;
+ @Override
public String toString() {
return getClass().getName() + "{task=" + AbstractTask.this + "}";
@@ -3016,6 +3019,7 @@
* Overridden to visit the name of all indices that were isolated and to
* ignore the timestamp.
*/
+ @SuppressWarnings("unchecked")
@Override
public Iterator<String> indexNameScan(final String prefix,
final long timestampIsIgnored) {
@@ -3054,7 +3058,8 @@
@SuppressWarnings("rawtypes")
private final DefaultResourceLocator resourceLocator;
-
+
+ @Override
public String toString() {
return getClass().getName() + "{task=" + AbstractTask.this + "}";
@@ -3313,7 +3318,8 @@
final IIndexManager tmp = new DelegateIndexManager(this) {
- public IIndex getIndex(String name, long timestampIsIgnored) {
+ @Override
+ public IIndex getIndex(final String name, final long timestampIsIgnored) {
// last commit time.
final long commitTime = delegate.getRootBlockView()
@@ -3610,7 +3616,7 @@
*/
private static class DelegateIndexManager implements IIndexManager {
- private IIndexManager delegate;
+ private final IIndexManager delegate;
public DelegateIndexManager(final IIndexManager delegate) {
this.delegate = delegate;
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IReadOnly.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IReadOnly.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/IReadOnly.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -0,0 +1,48 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Mar 15, 2007
+ */
+package com.bigdata.journal;
+
+/**
+ * A marker interface for logic that can declare whether or not it is read-only.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+public interface IReadOnly {
+
+ /**
+ * Return <code>true</code> iff the procedure asserts that it will not
+ * write on the index. When <code>true</code>, the procedure may be run
+ * against a view of the index that is read-only or which allows concurrent
+ * processes to read on the same index object. When <code>false</code> the
+ * procedure will be run against a mutable view of the index (assuming that
+ * the procedure is executed in a context that has access to a mutable index
+ * view).
+ */
+ public boolean isReadOnly();
+
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -39,7 +39,6 @@
* Implementation for a standalone journal using single-phase commits.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class JournalTransactionService extends
AbstractHATransactionService {
@@ -58,6 +57,7 @@
}
+ @Override
public JournalTransactionService start() {
super.start();
@@ -70,6 +70,7 @@
* Extended to register the new tx in the
* {@link AbstractLocalTransactionManager}.
*/
+ @Override
protected void activateTx(final TxState state) {
super.activateTx(state);
@@ -88,6 +89,7 @@
}
+ @Override
protected void deactivateTx(final TxState state) {
super.deactivateTx(state);
@@ -112,6 +114,7 @@
}
+ @Override
protected long findCommitTime(final long timestamp) {
final ICommitRecord commitRecord = journal.getCommitRecord(timestamp);
@@ -126,6 +129,7 @@
}
+ @Override
protected long findNextCommitTime(final long commitTime) {
/*
@@ -150,6 +154,7 @@
}
+ @Override
protected void abortImpl(final TxState state) {
if(state.isReadOnly()) {
@@ -220,6 +225,7 @@
}
+ @Override
protected long commitImpl(final TxState state) throws ExecutionException,
InterruptedException {
@@ -312,7 +318,7 @@
/*
* FIXME The state changes for the local tx should be atomic across
* this operation. In order to do that we have to make those changes
- * inside of SinglePhaseTask while it is holding the lock, but after
+ * inside of SinglePhaseCommit while it is holding the lock, but after
* it has committed. Perhaps the best way to do this is with a pre-
* and post- call() API since we can not hold the lock across the
* task otherwise (it will deadlock).
@@ -382,7 +388,6 @@
* coherent commit time for the transaction as a whole.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static class SinglePhaseCommit extends AbstractTask<Void> {
@@ -411,6 +416,7 @@
}
+ @Override
public Void doTask() throws Exception {
/*
@@ -446,6 +452,7 @@
/**
* The last commit time from the current root block.
*/
+ @Override
final public long getLastCommitTime() {
return journal.getRootBlockView().getLastCommitTime();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -139,25 +139,24 @@
* or a period of time then that might do it.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class WriteExecutorService extends ThreadPoolExecutor {
/**
- * Main log for the {@link WriteExecutorService}.
+ * Main logger for the {@link WriteExecutorService}.
*/
- protected static final Logger log = Logger
+ private static final Logger log = Logger
.getLogger(WriteExecutorService.class);
/**
* True iff the {@link #log} level is INFO or less.
*/
- final protected static boolean INFO = log.isInfoEnabled();
+ final private static boolean INFO = log.isInfoEnabled();
/**
* True iff the {@link #log} level is DEBUG or less.
*/
- final protected static boolean DEBUG = log.isDebugEnabled();
+ final private static boolean DEBUG = log.isDebugEnabled();
/**
* Uses the {@link OverflowManager} log for things relating to synchronous
@@ -453,13 +452,13 @@
* The threads that are running our tasks (so that we can interrupt them
* if necessary).
*/
- final private ConcurrentHashMap<Thread,AbstractTask> active = new ConcurrentHashMap<Thread,AbstractTask>();
+ final private ConcurrentHashMap<Thread,AbstractTask<?>> active = new ConcurrentHashMap<Thread,AbstractTask<?>>();
/**
* The set of tasks that make it into the commit group (so that we can set
* the commit time on each of them iff the group commit succeeds).
*/
- final private Map<Thread,AbstractTask> commitGroup = new LinkedHashMap<Thread, AbstractTask>();
+ final private Map<Thread,AbstractTask<?>> commitGroup = new LinkedHashMap<Thread, AbstractTask<?>>();
/** #of write tasks completed since the last commit. */
final private AtomicInteger nwrites = new AtomicInteger(0);
@@ -805,7 +804,7 @@
* @param r
* The {@link AbstractTask}.
*/
- protected void beforeTask(final Thread t, final AbstractTask r) {
+ protected void beforeTask(final Thread t, final AbstractTask<?> r) {
if (t == null)
throw new NullPointerException();
@@ -875,7 +874,7 @@
* The exception thrown -or- <code>null</code> if the task
* completed successfully.
*/
- protected void afterTask(final AbstractTask r, final Throwable t) {
+ protected void afterTask(final AbstractTask<?> r, final Throwable t) {
if (r == null)
throw new NullPointerException();
@@ -1043,7 +1042,7 @@
} finally {
// Remove since thread is no longer running the task.
- final ITask tmp = active.remove(Thread.currentThread());
+ final ITask<?> tmp = active.remove(Thread.currentThread());
if(trackActiveSetInMDC) {
@@ -1238,7 +1237,7 @@
final Thread currentThread = Thread.currentThread();
// the task that invoked this method.
- final ITask r = active.get(currentThread);
+ final ITask<?> r = active.get(currentThread);
/*
* If an abort is in progress then throw an exception.
@@ -2149,7 +2148,7 @@
* enough.]
*/
- final AbstractTask[] a = active.values().toArray(new AbstractTask[0]);
+ final AbstractTask<?>[] a = active.values().toArray(new AbstractTask[0]);
final TaskAndTime[] b = new TaskAndTime[a.length];
@@ -2195,7 +2194,7 @@
private static class TaskAndTime implements Comparable<TaskAndTime> {
private final long now;
- private final AbstractTask task;
+ private final AbstractTask<?> task;
/** The elapsed milliseconds for work performed on this task. */
private final long elapsedRunTime;
/** The #of milliseconds ago that work began on this task. */
@@ -2208,6 +2207,7 @@
}
private final State state;
+ @Override
public String toString() {
return "TaskAndTime{" + task.toString() + ",elapsedRunTime="
+ TimeUnit.NANOSECONDS.toMillis(elapsedRunTime)
@@ -2215,7 +2215,7 @@
+ ",state=" + state + "}";
}
- TaskAndTime(final AbstractTask task, final long now) {
+ TaskAndTime(final AbstractTask<?> task, final long now) {
this.task = task;
this.now = now;
if (task.nanoTime_finishedWork != 0L) {
@@ -2238,6 +2238,7 @@
/**
* Places into order by decreasing {@link #elapsedRunTime}.
*/
+ @Override
public int compareTo(final TaskAndTime o) {
if (elapsedRunTime < o.elapsedRunTime)
@@ -2417,7 +2418,7 @@
assert nwrites.get() == commitGroup.size();
- for (AbstractTask task : commitGroup.values()) {
+ for (AbstractTask<?> task : commitGroup.values()) {
task.commitTime = timestamp;
@@ -2543,14 +2544,14 @@
if (INFO)
log.info("Interrupting tasks awaiting commit.");
- final Iterator<Map.Entry<Thread, AbstractTask>> itr = active
+ final Iterator<Map.Entry<Thread, AbstractTask<?>>> itr = active
.entrySet().iterator();
int ninterrupted = 0;
while (itr.hasNext()) {
- final Map.Entry<Thread,AbstractTask> entry = itr.next();
+ final Map.Entry<Thread,AbstractTask<?>> entry = itr.next();
// set flag to deny access to resources.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -635,6 +635,7 @@
}
+ @Override
public long nextTimestamp() {
// setupLoggingContext();
@@ -698,6 +699,7 @@
* transactions which must contend for start times which will
* read from the appropriate historical commit point).
*/
+ @Override
public long newTx(final long timestamp) {
setupLoggingContext();
@@ -883,6 +885,7 @@
*
* @see Options#MIN_RELEASE_AGE
*/
+ @Override
public long getReleaseTime() {
if (log.isTraceEnabled())
@@ -1527,6 +1530,7 @@
* is invoked for all commits on all data services and will otherwise be a
* global hotspot.
*/
+ @Override
public void notifyCommit(final long commitTime) {
lock.lock();
@@ -1930,6 +1934,7 @@
/**
* Note: Declared abstract so that we can hide the {@link IOException}.
*/
+ @Override
abstract public long getLastCommitTime();
/**
@@ -2008,6 +2013,7 @@
/**
* Abort the transaction (asynchronous).
*/
+ @Override
public void abort(final long tx) {
setupLoggingContext();
@@ -2097,6 +2103,7 @@
}
+ @Override
public long commit(final long tx) throws ValidationError {
setupLoggingContext();
@@ -2475,6 +2482,7 @@
/**
* The hash code is based on the {@link #getStartTimestamp()}.
*/
+ @Override
final public int hashCode() {
return hashCode;
@@ -2489,6 +2497,7 @@
* @param o
* Another transaction object.
*/
+ @Override
final public boolean equals(final Object o) {
if (this == o)
@@ -2607,6 +2616,7 @@
/**
* Returns a string representation of the transaction state.
*/
+ @Override
final public String toString() {
/*
@@ -2691,6 +2701,7 @@
* {@link #getLastCommitTime()} and then changes the {@link TxServiceRunState}
* to {@link TxServiceRunState#Running}.
*/
+ @Override
public AbstractTransactionService start() {
if(log.isInfoEnabled())
@@ -2741,6 +2752,7 @@
}
+ @Override
@SuppressWarnings("rawtypes")
public Class getServiceIface() {
@@ -2758,60 +2770,70 @@
final CounterSet countersRoot = new CounterSet();
countersRoot.addCounter("runState", new Instrument<String>() {
+ @Override
protected void sample() {
setValue(runState.toString());
}
});
countersRoot.addCounter("#active", new Instrument<Integer>() {
+ @Override
protected void sample() {
setValue(getActiveCount());
}
});
countersRoot.addCounter("lastCommitTime", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getLastCommitTime());
}
});
countersRoot.addCounter("minReleaseAge", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getMinReleaseAge());
}
});
countersRoot.addCounter("releaseTime", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getReleaseTime());
}
});
countersRoot.addCounter("startCount", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getStartCount());
}
});
countersRoot.addCounter("abortCount", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getAbortCount());
}
});
countersRoot.addCounter("commitCount", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getCommitCount());
}
});
countersRoot.addCounter("readOnlyActiveCount", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getReadOnlyActiveCount());
}
});
countersRoot.addCounter("readWriteActiveCount", new Instrument<Long>() {
+ @Override
protected void sample() {
setValue(getReadWriteActiveCount());
}
@@ -2826,6 +2848,7 @@
*/
countersRoot.addCounter("earliestReadsOnCommitTime",
new Instrument<Long>() {
+ @Override
protected void sample() {
final TxState tmp = earliestOpenTx;
if (tmp != null)
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentTx.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentTx.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentTx.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -95,27 +95,29 @@
* index that result in write-write conflicts.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
-public class StressTestConcurrentTx extends ProxyTestCase implements IComparisonTest {
+public class StressTestConcurrentTx extends ProxyTestCase<Journal> implements
+ IComparisonTest {
public StressTestConcurrentTx() {
}
- public StressTestConcurrentTx(String name) {
+ public StressTestConcurrentTx(final String name) {
super(name);
}
- Journal journal;
+ private Journal journal;
- public void setUpComparisonTest(Properties properties) throws Exception {
+ @Override
+ public void setUpComparisonTest(final Properties properties) throws Exception {
journal = new Journal(properties);
}
+ @Override
public void tearDownComparisonTest() throws Exception {
if (journal != null) {
@@ -160,11 +162,13 @@
100,// nops
.10// abortRate
);
+
} finally {
journal.destroy();
}
+
}
/**
@@ -405,9 +409,9 @@
private final int nops;
private final double abortRate;
- final Random r = new Random();
+ private final Random r = new Random();
- public Task(Journal journal,String name, int trial, int keyLen, int nops, double abortRate) {
+ public Task(final Journal journal,final String name, int trial, int keyLen, int nops, double abortRate) {
this.journal = journal;
@@ -423,6 +427,7 @@
}
+ @Override
public String toString() {
return super.toString()+"#"+trial;
@@ -435,6 +440,7 @@
* @return The commit time of the transactions and <code>0L</code> IFF
* the transaction was aborted.
*/
+ @Override
public Long call() throws Exception {
final long tx = journal.newTx(ITx.UNISOLATED);
@@ -445,8 +451,9 @@
* complete.
*/
- journal.submit(new AbstractTask(journal, tx, name) {
+ journal.submit(new AbstractTask<Object>(journal, tx, name) {
+ @Override
protected Object doTask() {
// Random operations on the named index(s).
@@ -532,9 +539,9 @@
* @see GenerateExperiment, which may be used to generate a set of
* conditions to be run by the {@link ExperimentDriver}.
*/
- public static void main(String[] args) throws Exception {
+ public static void main(final String[] args) throws Exception {
- Properties properties = new Properties();
+ final Properties properties = new Properties();
// properties.setProperty(Options.FORCE_ON_COMMIT,ForceEnum.No.toString());
@@ -560,7 +567,7 @@
properties.setProperty(TestOptions.ABORT_RATE,".05");
- IComparisonTest test = new StressTestConcurrentTx();
+ final IComparisonTest test = new StressTestConcurrentTx();
test.setUpComparisonTest(properties);
@@ -632,7 +639,8 @@
* There are no "optional" properties - you must make sure that
* each property has a defined value.
*/
- public Result doComparisonTest(Properties properties) throws Exception {
+ @Override
+ public Result doComparisonTest(final Properties properties) throws Exception {
final long timeout = Long.parseLong(properties.getProperty(TestOptions.TIMEOUT));
@@ -670,16 +678,16 @@
*
* @param args
*/
- public static void main(String[] args) throws Exception {
+ public static void main(final String[] args) throws Exception {
// this is the test to be run.
- String className = StressTestConcurrentTx.class.getName();
+ final String className = StressTestConcurrentTx.class.getName();
/*
* Set defaults for each condition.
*/
- Map<String,String> defaultProperties = new HashMap<String,String>();
+ final Map<String,String> defaultProperties = new HashMap<String,String>();
// force delete of the files on close of the journal under test.
defaultProperties.put(Options.CREATE_TEMP_FILE,"true");
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentUnisolatedIndices.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentUnisolatedIndices.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestConcurrentUnisolatedIndices.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -75,6 +75,7 @@
private Journal journal;
+ @Override
public void setUpComparisonTest(final Properties properties)
throws Exception {
@@ -82,6 +83,7 @@
}
+ @Override
public void tearDownComparisonTest() throws Exception {
if (journal != null) {
@@ -425,6 +427,7 @@
}
+ @Override
protected String getTaskName() {
return super.getTaskName()+"#"+trial;
@@ -436,6 +439,7 @@
*
* @return null
*/
+ @Override
public Object doTask() throws Exception {
// the index names on which the writer holds a lock.
@@ -698,6 +702,7 @@
* There are no "optional" properties - you must make sure that
* each property has a defined value.
*/
+ @Override
public Result doComparisonTest(Properties properties) throws Exception {
final long timeout = Long.parseLong(properties.getProperty(TestOptions.TIMEOUT));
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestLockContention.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestLockContention.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestLockContention.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -48,9 +48,8 @@
* should still block many serialized writer tasks together for good throughput.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
-public class StressTestLockContention extends ProxyTestCase {
+public class StressTestLockContention extends ProxyTestCase<Journal> {
/**
*
@@ -62,7 +61,7 @@
/**
* @param name
*/
- public StressTestLockContention(String name) {
+ public StressTestLockContention(final String name) {
super(name);
}
@@ -88,7 +87,7 @@
final int ntasks = 500;
- final List<Future> futures;
+ final List<Future<Object>> futures;
{
@@ -100,13 +99,14 @@
final String[] resource = new String[] { "foo", "bar", "baz" };
- final Collection<AbstractTask> tasks = new HashSet<AbstractTask>(
+ final Collection<AbstractTask<Object>> tasks = new HashSet<AbstractTask<Object>>(
ntasks);
for (int i = 0; i < ntasks; i++) {
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, resource) {
+ tasks.add(new AbstractTask<Object>(journal, ITx.UNISOLATED, resource) {
+ @Override
protected Object doTask() throws Exception {
return null;
@@ -124,7 +124,7 @@
* returns.
*/
- futures = journal.invokeAll(tasks, 20, TimeUnit.SECONDS);
+ futures = (List)journal.invokeAll(tasks, 20, TimeUnit.SECONDS);
} finally {
@@ -153,7 +153,7 @@
}
- final Iterator<Future> itr = futures.iterator();
+ final Iterator<Future<Object>> itr = futures.iterator();
int ncancelled = 0;
int ncomplete = 0;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -56,7 +56,6 @@
* {@link UnisolatedReadWriteIndex} rather than the {@link ConcurrencyManager}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
* @see UnisolatedReadWriteIndex
*/
public class StressTestUnisolatedReadWriteIndex extends ProxyTestCase<Journal> {
@@ -435,6 +434,7 @@
}
+ @Override
public String toString() {
return getClass().getName() + "#" + trial;
@@ -446,6 +446,7 @@
*
* @return null
*/
+ @Override
public Void call() throws Exception {
final IIndex[] indices = new IIndex[resource.length];
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -286,6 +286,13 @@
}
@Override
+ public boolean isReadOnly() {
+
+ return true;
+
+ }
+
+ @Override
public TupleQueryResult call() throws Exception {
BigdataSailRepositoryConnection cxn = null;
boolean success = false;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/AbstractApiTask.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -36,6 +36,7 @@
import com.bigdata.BigdataStatics;
import com.bigdata.journal.IConcurrencyManager;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.journal.IReadOnly;
import com.bigdata.journal.ITx;
import com.bigdata.journal.Journal;
import com.bigdata.journal.TimestampUtility;
@@ -57,7 +58,7 @@
* @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
* Concurrent unisolated operations against multiple KBs </a>
*/
-abstract public class AbstractApiTask<T> implements IApiTask<T> {
+abstract public class AbstractApiTask<T> implements IApiTask<T>, IReadOnly {
/** The reference to the {@link IIndexManager} is set before the task is executed. */
private final AtomicReference<IIndexManager> indexManagerRef = new AtomicReference<IIndexManager>();
@@ -69,6 +70,9 @@
protected final long timestamp;
@Override
+ abstract public boolean isReadOnly();
+
+ @Override
public String getNamespace() {
return namespace;
}
@@ -119,28 +123,21 @@
}
/**
- * Return a read-only view of the {@link AbstractTripleStore} for the given
- * namespace will read from the commit point associated with the given
- * timestamp.
+ * Return a view of the {@link AbstractTripleStore} for the given namespace
+ * that will read on the commit point associated with the given timestamp.
*
* @param namespace
* The namespace.
* @param timestamp
- * The timestamp.
+ * The timestamp or {@link ITx#UNISOLATED} to obtain a read/write
+ * view of the index.
*
* @return The {@link AbstractTripleStore} -or- <code>null</code> if none is
* found for that namespace and timestamp.
- *
- * TODO Enforce historical query by making sure timestamps conform
- * (we do not want to allow read/write tx queries unless update
- * semantics are introduced ala SPARQL 1.1).
*/
protected AbstractTripleStore getTripleStore(final String namespace,
final long timestamp) {
- // if (timestamp == ITx.UNISOLATED)
- // throw new IllegalArgumentException("UNISOLATED reads disallowed.");
-
// resolve the default namespace.
final AbstractTripleStore tripleStore = (AbstractTripleStore) getIndexManager()
.getResourceLocator().locate(namespace, timestamp);
@@ -275,7 +272,9 @@
* Execute the REST API task.
*
* Note: For scale-out, the operation will be applied using
- * client-side global views of the indices.
+ * client-side global views of the indices. This means that
+ * there will not be any globally consistent views of the
+ * indices and that updates will be shard-wise local.
*
* Note: This can be used for operations on read-only views (even on
* a Journal). This is helpful since we can avoid some overhead
@@ -328,6 +327,11 @@
* namespace.
*
* (b) is now possible with the fix to the Name2Addr prefix scan.
+ *
+ * Note: We also need to isolate any named solution sets in the
+ * namespace of the KB. Those will be discovered along with the
+ * indices, but they may require changes to {@link AbstractTask}
+ * for GIST support.
*/
// Obtain the necessary locks for R/w access to KB indices.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/ApiTaskForIndexManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/ApiTaskForIndexManager.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/task/ApiTaskForIndexManager.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -37,6 +37,9 @@
* view of a scale-out index. It can also be used for a {@link Journal} if are
* not relying on the {@link IConcurrencyManager} to guard the resources
* declared by the task.
+ * <p>
+ * Note: Global locks are NOT used in scale-out and operations will be only
+ * shard-wise ACID.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @param <T>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/AbstractRestApiTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/AbstractRestApiTask.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/AbstractRestApiTask.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -41,14 +41,6 @@
* doLocalAbort() should interrupt NSS requests and AbstractTasks </a>
* @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
* Concurrent unisolated operations against multiple KBs </a>
- *
- * FIXME GROUP COMMIT: Define DropKBTask and CreateKBTask for use by (a)
- * the multi-tenancy API; and (b) variants without servlet request and
- * response parameters for use by the unit tests and the NSS during its
- * default KB create logic. These latter tasks should be a base class of
- * the RestApiTask that supports the same delegation pattern, but which
- * does not require the http request and response parameters. Fix up the
- * callers of CreateKBTask and tripleStore.destroy() to use these tasks.
*/
abstract class AbstractRestApiTask<T> extends AbstractApiTask<T> {
@@ -82,50 +74,57 @@
}
- abstract static class RestApiQueryTask<T> extends AbstractRestApiTask<T> {
- /**
- *
- * @param req
- * The {@link HttpServletRequest}.
- * @param resp
- * The {@link HttpServletResponse}.
- * @param namespace
- * The namespace of the target KB instance.
- * @param timestamp
- * The timestamp used to obtain a query connection.
- */
- public RestApiQueryTask(//
- final HttpServletRequest req,//
- final HttpServletResponse resp,//
- final String namespace, final long timestamp) {
-
- super(req, resp, namespace, timestamp);
-
- }
-
- }
-
- abstract static class RestApiMutationTask<T> extends AbstractRestApiTask<T> {
- /**
- *
- * @param req
- * The {@link HttpServletRequest}.
- * @param resp
- * The {@link HttpServletResponse}.
- * @param namespace
- * The namespace of the target KB instance.
- * @param timestamp
- * The timestamp used to obtain a mutable connection.
- */
- public RestApiMutationTask(//
- final HttpServletRequest req,//
- final HttpServletResponse resp,//
- final String namespace, final long timestamp) {
-
- super(req, resp, namespace, timestamp);
-
- }
-
- }
+ /*
+ * Note: The introduction of a read/write task distinction at this level is
+ * problematic because some tasks must be declared in terms of the
+ * AbstractApiTask and then add the servlet parameters. Thus, this
+ * distinction could only really be captured through a marker interface
+ * (such as we have for
+ */
+// abstract static class RestApiQueryTask<T> extends AbstractRestApiTask<T> {
+// /**
+// *
+// * @param req
+// * The {@link HttpServletRequest}.
+// * @param resp
+// * The {@link HttpServletResponse}.
+// * @param namespace
+// * The namespace of the target KB instance.
+// * @param timestamp
+// * The timestamp used to obtain a query connection.
+// */
+// public RestApiQueryTask(//
+// final HttpServletRequest req,//
+// final HttpServletResponse resp,//
+// final String namespace, final long timestamp) {
+//
+// super(req, resp, namespace, timestamp);
+//
+// }
+//
+// }
+//
+// abstract static class RestApiMutationTask<T> extends AbstractRestApiTask<T> {
+// /**
+// *
+// * @param req
+// * The {@link HttpServletRequest}.
+// * @param resp
+// * The {@link HttpServletResponse}.
+// * @param namespace
+// * The namespace of the target KB instance.
+// * @param timestamp
+// * The timestamp used to obtain a mutable connection.
+// */
+// public RestApiMutationTask(//
+// final HttpServletRequest req,//
+// final HttpServletResponse resp,//
+// final String namespace, final long timestamp) {
+//
+// super(req, resp, namespace, timestamp);
+//
+// }
+//
+// }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -95,7 +95,6 @@
import com.bigdata.rdf.sail.ISPARQLUpdateListener;
import com.bigdata.rdf.sail.SPARQLUpdateEvent;
import com.bigdata.rdf.sail.sparql.Bigdata2ASTSPARQLParser;
-import com.bigdata.rdf.sail.webapp.AbstractRestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.client.StringUtil;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.QueryHints;
@@ -1154,6 +1153,14 @@
}
@Override
+ public boolean isReadOnly() {
+
+ // Read-only unless SPARQL UPDATE.
+ return !AbstractQueryTask.this.update;
+
+ }
+
+ @Override
public Void call() throws Exception {
BigdataSailRepositoryConnection cxn = null;
boolean success = false;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -33,7 +33,6 @@
import com.bigdata.blueprints.BigdataGraphBulkLoad;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
-import com.bigdata.rdf.sail.webapp.AbstractRestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.client.MiniMime;
import com.tinkerpop.blueprints.util.io.graphml.GraphMLReader;
@@ -108,7 +107,7 @@
}
- private static class BlueprintsPostTask extends RestApiMutationTask<Void> {
+ private static class BlueprintsPostTask extends AbstractRestApiTask<Void> {
public BlueprintsPostTask(HttpServletRequest req,
HttpServletResponse resp, String namespace, long timestamp) {
@@ -118,6 +117,11 @@
}
@Override
+ public boolean isReadOnly() {
+ return false;
+ }
+
+ @Override
public Void call() throws Exception {
final long begin = System.currentTimeMillis();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -48,7 +48,6 @@
import com.bigdata.journal.ITx;
import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
-import com.bigdata.rdf.sail.webapp.AbstractRestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask;
import com.bigdata.rdf.sail.webapp.client.EncodeDecodeValue;
import com.bigdata.rdf.sail.webapp.client.MiniMime;
@@ -348,7 +347,7 @@
}
- private static class DeleteWithBodyTask extends RestApiMutationTask<Void> {
+ private static class DeleteWithBodyTask extends AbstractRestApiTask<Void> {
private final String baseURI;
private final Resource[] defaultContext;
@@ -382,6 +381,11 @@
}
@Override
+ public boolean isReadOnly() {
+ return false;
+ }
+
+ @Override
public Void call() throws Exception {
final long begin = System.currentTimeMillis();
@@ -545,7 +549,7 @@
// static private transient final Resource[] nullArray = new Resource[]{};
- private static class DeleteWithAccessPathTask extends RestApiMutationTask<Void> {
+ private static class DeleteWithAccessPathTask extends AbstractRestApiTask<Void> {
private Resource s;
private URI p;
@@ -580,6 +584,11 @@
}
@Override
+ public boolean isReadOnly() {
+ return true;
+ }
+
+ @Override
public Void call() throws Exception {
final long begin = System.currentTimeMillis();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -49,7 +49,6 @@
import com.bigdata.rdf.rio.IRDFParserOptions;
import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
-import com.bigdata.rdf.sail.webapp.AbstractRestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.client.MiniMime;
/**
@@ -224,7 +223,7 @@
* TODO The {@link IRDFParserOptions} defaults should be coming from
* the KB instance, right? What does the REST API say about this?
*/
- private static class InsertWithBodyTask extends RestApiMutationTask<Void> {
+ private static class InsertWithBodyTask extends AbstractRestApiTask<Void> {
private final String baseURI;
private final Resource[] defaultContext;
@@ -258,6 +257,11 @@
}
@Override
+ public boolean isReadOnly() {
+ return false;
+ }
+
+ @Override
public Void call() throws Exception {
final long begin = System.currentTimeMillis();
@@ -407,7 +411,7 @@
}
- private static class InsertWithURLsTask extends RestApiMutationTask<Void> {
+ private static class InsertWithURLsTask extends AbstractRestApiTask<Void> {
private final Vector<URL> urls;
private final Resource[] defaultContext;
@@ -437,6 +441,11 @@
}
@Override
+ public boolean isReadOnly() {
+ return false;
+ }
+
+ @Override
public Void call() throws Exception {
final long begin = System.currentTimeMillis();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -65,8 +65,14 @@
* @author thompsonbry
*
* FIXME GROUP COMMIT: The CREATE and DESTROY operations require special
- * attention. The other operations in this class also should use the new
- * REST API pattern, but are not intrinsically sensitive.
+ * attention. Define DropKBTask and CreateKBTask for use by the
+ * multi-tenancy API and fix up the callers of CreateKBTask and
+ * tripleStore.destroy() to use these tasks. This means that the base
+ * implementations of those tasks must not require the servlet
+ * parameters.
+ *
+ * FIXME GROUP COMMIT: The other operations in this class also should
+ * use the new REST API pattern, but are not intrinsically sensitive.
*/
public class MultiTenancyServlet extends BigdataRDFServlet {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2014-07-16 14:52:28 UTC (rev 8553)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2014-07-16 14:56:31 UTC (rev 8554)
@@ -66,7 +66,6 @@
import com.bigdata.rdf.sail.BigdataSailQuery;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
import com.bigdata.rdf.sail.sparql.ast.SimpleNode;
-import com.bigdata.rdf.sail.webapp.AbstractRestApiTask.RestApiQueryTask;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.RunningQuery;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.UpdateTask;
@@ -1073,7 +1072,7 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
- private static class EstCardTask extends RestApiQueryTask<Void> {
+ private static class EstCardTask extends AbstractRestApiTask<Void> {
private final Resource s;
private final URI p;
@@ -1093,6 +1092,11 @@
...
[truncated message content] |
|
From: <tho...@us...> - 2014-07-16 14:52:36
|
Revision: 8553
http://sourceforge.net/p/bigdata/code/8553
Author: thompsonbry
Date: 2014-07-16 14:52:28 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
test suite cleanup
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestWORMStrategy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestWORMStrategy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestWORMStrategy.java 2014-07-16 14:51:18 UTC (rev 8552)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestWORMStrategy.java 2014-07-16 14:52:28 UTC (rev 8553)
@@ -27,30 +27,20 @@
package com.bigdata.journal;
-import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.nio.ByteBuffer;
import java.util.Properties;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
import junit.extensions.proxy.ProxyTestSuite;
import junit.framework.Test;
import com.bigdata.io.DirectBufferPool;
-import com.bigdata.rawstore.IPSOutputStream;
import com.bigdata.rawstore.IRawStore;
/**
* Test suite for {@link WORMStrategy} journal.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @see TestWORMStrategyNoCache
* @see TestWORMStrategyOneCacheBuffer
@@ -106,6 +96,7 @@
}
+ @Override
public Properties getProperties() {
final Properties properties = super.getProperties();
@@ -207,7 +198,6 @@
* Test suite integration for {@link AbstractRestartSafeTestCase}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static class TestRawStore extends AbstractRestartSafeTestCase {
@@ -219,6 +209,7 @@
super(name);
}
+ @Override
protected BufferMode getBufferMode() {
return BufferMode.DiskWORM;
@@ -231,7 +222,6 @@
* Test suite integration for {@link AbstractInterruptsTestCase}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static class TestInterrupts extends AbstractInterruptsTestCase {
@@ -243,6 +233,7 @@
super(name);
}
+ @Override
protected IRawStore getStore() {
final Properties properties = getProperties();
@@ -268,7 +259,6 @@
* Test suite integration for {@link AbstractMROWTestCase}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static class TestMROW extends AbstractMROWTestCase {
@@ -280,6 +270,7 @@
super(name);
}
+ @Override
protected Journal getStore() {
final Properties properties = getProperties();
@@ -304,7 +295,6 @@
* Test suite integration for {@link AbstractMRMWTestCase}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static class TestMRMW extends AbstractMRMWTestCase {
@@ -316,6 +306,7 @@
super(name);
}
+ @Override
protected IRawStore getStore() {
final Properties properties = getProperties();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-16 14:51:21
|
Revision: 8552
http://sourceforge.net/p/bigdata/code/8552
Author: thompsonbry
Date: 2014-07-16 14:51:18 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
test suite cleanup
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestUnisolatedWriteTasks.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestUnisolatedWriteTasks.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestUnisolatedWriteTasks.java 2014-07-16 14:49:08 UTC (rev 8551)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestUnisolatedWriteTasks.java 2014-07-16 14:51:18 UTC (rev 8552)
@@ -39,7 +39,7 @@
/**
* Correctness test suite for unisolated writes on one or more indices. The
- * tests in this suite validate that the unisolated writes resulting in a commit
+ * tests in this suite validate that the unisolated writes result in a commit
* and that the committed data may be read back by an unisolated read task. Some
* tests write on more than one index in order to verify that the writes and
* reads are performed against the expected index. The stress test additionally
@@ -47,7 +47,6 @@
* list and that writes are NOT lost in large commit groups.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class TestUnisolatedWriteTasks extends ProxyTestCase<Journal> {
@@ -61,7 +60,7 @@
/**
* @param name
*/
- public TestUnisolatedWriteTasks(String name) {
+ public TestUnisolatedWriteTasks(final String name) {
super(name);
}
@@ -117,15 +116,18 @@
//
// journal.submit(
- new AbstractTask(journal,ITx.UNISOLATED,resource) {
+ new AbstractTask<Object>(journal,ITx.UNISOLATED,resource) {
+ @Override
protected Object doTask() throws Exception {
- log.info("Will register " + getOnlyResource());
+ if(log.isInfoEnabled())
+ log.info("Will register " + getOnlyResource());
getJournal().registerIndex(getOnlyResource(),new IndexMetadata(resource[0], indexUUID));
- log.info("Will write on " + getOnlyResource());
+ if(log.isInfoEnabled())
+ log.info("Will write on " + getOnlyResource());
final IIndex ndx = getIndex(getOnlyResource());
@@ -158,12 +160,14 @@
* Submit task to validate the committed unisolated write on the named
* index.
*/
- journal.submit(new AbstractTask(journal, ITx.READ_COMMITTED,
+ journal.submit(new AbstractTask<Object>(journal, ITx.READ_COMMITTED,
resource) {
+ @Override
protected Object doTask() throws Exception {
- log.info("Will read from " + getOnlyResource());
+ if(log.isInfoEnabled())
+ log.info("Will read from " + getOnlyResource());
final IIndex ndx = getIndex(getOnlyResource());
@@ -264,8 +268,9 @@
}
// submit task to create index and do batch insert on that index.
- journal.submit(new AbstractTask(journal, ITx.UNISOLATED, resource) {
+ journal.submit(new AbstractTask<Object>(journal, ITx.UNISOLATED, resource) {
+ @Override
protected Object doTask() throws Exception {
getJournal().registerIndex(resource[0], //
@@ -315,13 +320,14 @@
* Submit task to validate the committed unisolated write on the named
* index.
*/
- journal.submit(new AbstractTask(journal, ITx.READ_COMMITTED, resource) {
+ journal.submit(new AbstractTask<Object>(journal, ITx.READ_COMMITTED, resource) {
+ @Override
protected Object doTask() throws Exception {
{
- IIndex ndx = getIndex("foo");
+ final IIndex ndx = getIndex("foo");
assertEquals("indexUUID", indexUUID1, ndx
.getIndexMetadata().getIndexUUID());
@@ -500,8 +506,9 @@
// submit task to create indices and do batch inserts those indices.
journal.submit(
- new AbstractTask(journal, ITx.UNISOLATED,resource) {
+ new AbstractTask<Object>(journal, ITx.UNISOLATED,resource) {
+ @Override
protected Object doTask() throws Exception {
// register all indices.
@@ -549,9 +556,10 @@
* index.
*/
journal.submit(
- new AbstractTask(journal, ITx.READ_COMMITTED, resource) {
+ new AbstractTask<Object>(journal, ITx.READ_COMMITTED, resource) {
// verify the writes on each index.
+ @Override
protected Object doTask() throws Exception {
for(int i=0; i<nindices; i++) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-16 14:49:18
|
Revision: 8551
http://sourceforge.net/p/bigdata/code/8551
Author: thompsonbry
Date: 2014-07-16 14:49:08 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
javadoc related to #343
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-07-16 14:48:52 UTC (rev 8550)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-07-16 14:49:08 UTC (rev 8551)
@@ -3623,7 +3623,7 @@
assert node.dirty;
assert !node.deleted;
assert !node.isPersistent();
- assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. See https://sourceforge.net/apps/trac/bigdata/ticket/343
+ assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
assertNotReadOnly();
/*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 8550
http://sourceforge.net/p/bigdata/code/8550
Author: thompsonbry
Date: 2014-07-16 14:48:52 +0000 (Wed, 16 Jul 2014)
Log Message:
-----------
javadoc cleanup
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestNanoSparqlServerWithProxyIndexManager.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestNanoSparqlServerWithProxyIndexManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestNanoSparqlServerWithProxyIndexManager.java 2014-07-15 13:01:35 UTC (rev 8549)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestNanoSparqlServerWithProxyIndexManager.java 2014-07-16 14:48:52 UTC (rev 8550)
@@ -60,23 +60,7 @@
* queries which exercise the context position; and the default-graph and
* named-graph URL query parameters for quads.
*
- * @todo Security model?
- *
- * @todo An NQUADS RDFWriter needs to be written. Then we can test NQUADS
- * interchange.
- *
- * @todo A SPARQL result sets JSON parser needs to be written (Sesame bundles a
- * writer, but not a parser) before we can test queries which CONNEG for a
- * JSON result set.
- *
- * @todo Tests which verify the correct rejection of illegal or ill-formed
- * requests.
- *
- * @todo Test suite for reading from a historical commit point.
- *
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: TestNanoSparqlServer.java 4398 2011-04-14 13:55:29Z thompsonbry
- * $
*/
public class TestNanoSparqlServerWithProxyIndexManager<S extends IIndexManager>
extends AbstractIndexManagerTestCase<S> {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-15 13:01:43
|
Revision: 8549
http://sourceforge.net/p/bigdata/code/8549
Author: thompsonbry
Date: 2014-07-15 13:01:35 +0000 (Tue, 15 Jul 2014)
Log Message:
-----------
Bug fix to the query deadline support. The remaining time until the next check of the deadline queue was computed incorrectly. This resulted in a progressive increase in the time until the next deadline check.
The logic in QueryEngine.QueryTask.run() to check the deadline queue has been fixed.
The code has been converted to use nanoseconds rather than milliseconds for checking deadlines.
The deadline check code now runs every 100ms, as intended. We should recheck hot query performance for large concurrent query workloads to ensure that this does not translate into a query penalty (it should not as the code is polling on the query chunks available for processing and will grab a check before 100ms has expired if there is one available).
See #242 (Deadlines do not play well with GROUP_BY, ORDER_BY, etc.)
See #772 (Query timeout only checked at operator start/stop)
See #865 (OutOfMemoryError instead of Timeout for SPARQL Property Paths)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -99,7 +99,6 @@
* first result when compared with pipelined evaluation.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class AbstractRunningQuery implements IRunningQuery {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -114,8 +114,8 @@
Map<Integer/* bopId */, BOpStats> getStats();
/**
- * Return the query deadline (the time at which it will terminate regardless
- * of its run state).
+ * Return the query deadline in milliseconds (the time at which it will
+ * terminate regardless of its run state).
*
* @return The query deadline (milliseconds since the epoch) and
* {@link Long#MAX_VALUE} if no explicit deadline was specified.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -19,9 +19,9 @@
class QueryDeadline implements Comparable<QueryDeadline> {
/**
- * The deadline for this query.
+ * The deadline for this query (in nanoseconds).
*/
- final long deadline;
+ final long deadlineNanos;
/**
* A reference to the query.
@@ -33,14 +33,14 @@
/**
*
- * @param deadline
- * The deadline.
+ * @param deadlineNanos
+ * The deadline for this query (in nanoseconds).
* @param query
* The query.
*/
- public QueryDeadline(final long deadline, final AbstractRunningQuery query) {
+ public QueryDeadline(final long deadlineNanos, final AbstractRunningQuery query) {
- this.deadline = deadline;
+ this.deadlineNanos = deadlineNanos;
this.queryRef = new WeakReference<AbstractRunningQuery>(query);
@@ -61,8 +61,8 @@
*/
@Override
public int compareTo(final QueryDeadline o) {
- final long d0 = this.deadline;
- final long d1 = o.deadline;
+ final long d0 = this.deadlineNanos;
+ final long d1 = o.deadlineNanos;
if (d0 < d1)
return -1;
if (d0 > d1)
@@ -74,13 +74,13 @@
* Check the deadline on the query. If the query is not terminated and the
* deadline has expired, then the query is terminated as a side-effect.
*
- * @param now
+ * @param nowNanosIsIgnored
* A current timestamp.
*
* @return <code>null</code> if the query is terminated and
* <code>this</code> if the query is not terminated.
*/
- QueryDeadline checkDeadline(final long now) {
+ QueryDeadline checkDeadline(final long nowNanosIsIgnored) {
final AbstractRunningQuery q = queryRef.get();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -195,7 +195,6 @@
* query manager task for the terminated join.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo Expander patterns will continue to exist until we handle the standalone
* backchainers in a different manner for scale-out so add support for
@@ -669,15 +668,17 @@
throw new IllegalArgumentException();
}
- deadlineQueue.add(new QueryDeadline(deadline, query));
+ final long deadlineNanos = TimeUnit.MILLISECONDS.toNanos(deadline);
+ deadlineQueue.add(new QueryDeadline(deadlineNanos, query));
+
}
/**
* Scan the priority queue of queries with a specified deadline, halting any
* queries whose deadline has expired.
*/
- static private void checkDeadlines(final long now,
+ static private void checkDeadlines(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
/*
@@ -690,7 +691,7 @@
* Check the head of the deadline queue for any queries whose
* deadline has expired.
*/
- checkHeadOfDeadlineQueue(now, deadlineQueue);
+ checkHeadOfDeadlineQueue(nowNanos, deadlineQueue);
if (deadlineQueue.size() > DEADLINE_QUEUE_SCAN_SIZE) {
@@ -698,7 +699,7 @@
* Scan the deadline queue, removing entries for expired
* queries.
*/
- scanDeadlineQueue(now, deadlineQueue);
+ scanDeadlineQueue(nowNanos, deadlineQueue);
}
@@ -710,7 +711,7 @@
* Check the head of the deadline queue for any queries whose deadline has
* expired.
*/
- static private void checkHeadOfDeadlineQueue(final long now,
+ static private void checkHeadOfDeadlineQueue(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
QueryDeadline x;
@@ -719,7 +720,7 @@
while ((x = deadlineQueue.poll()) != null) {
// test for query done or deadline expired.
- if (x.checkDeadline(now) == null) {
+ if (x.checkDeadline(nowNanos) == null) {
/*
* This query is known to be done. It was removed from the
@@ -731,7 +732,7 @@
}
- if (x.deadline > now) {
+ if (x.deadlineNanos > nowNanos) {
/*
* This query has not yet reached its deadline. That means that
@@ -757,7 +758,7 @@
* has not be reached. Therefore, periodically, we need to scan the queue
* and clear out entries for terminated queries.
*/
- static private void scanDeadlineQueue(final long now,
+ static private void scanDeadlineQueue(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
final List<QueryDeadline> c = new ArrayList<QueryDeadline>(
@@ -770,7 +771,7 @@
for (QueryDeadline x : c) {
- if (x.checkDeadline(now) != null) {
+ if (x.checkDeadline(nowNanos) != null) {
// return this query to the deadline queue.
deadlineQueue.add(x);
@@ -939,27 +940,31 @@
if(log.isInfoEnabled())
log.info("Running: " + this);
try {
- long mark = System.currentTimeMillis();
- long remaining = DEADLINE_CHECK_MILLIS;
+ final long deadline = TimeUnit.MILLISECONDS
+ .toNanos(DEADLINE_CHECK_MILLIS);
+ long mark = System.nanoTime();
+ long remaining = deadline;
while (true) {
try {
+ //log.warn("Polling deadline queue: remaining="+remaining+", deadlinkCheckMillis="+DEADLINE_CHECK_MILLIS);
final AbstractRunningQuery q = priorityQueue.poll(
- remaining, TimeUnit.MILLISECONDS);
- final long now = System.currentTimeMillis();
- if ((remaining = now - mark) < 0) {
+ remaining, TimeUnit.NANOSECONDS);
+ final long now = System.nanoTime();
+ if ((remaining = deadline - (now - mark)) < 0) {
+ //log.error("Checking deadline queue");
/*
* Check for queries whose deadline is expired.
- *
+ *
* Note: We only do this every DEADLINE_CHECK_MILLIS
* and then reset [mark] and [remaining].
- *
+ *
* Note: In queue.pool(), we only wait only up to
* the [remaining] time before the next check in
* queue.poll().
*/
checkDeadlines(now, deadlineQueue);
mark = now;
- remaining = DEADLINE_CHECK_MILLIS;
+ remaining = deadline;
}
// Consume chunk already on queue for this query.
if (q != null && !q.isDone())
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -25,6 +25,7 @@
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import junit.framework.TestCase2;
@@ -155,20 +156,26 @@
final AbstractRunningQuery runningQuery1 = queryEngine.eval(UUID.randomUUID(),
query1, new ListBindingSet());
- runningQuery1.setDeadline(now + 10000);
+ final long deadline1Millis = now + 10000/* millis */;
+ runningQuery1.setDeadline(deadline1Millis);
+
Thread.sleep(2);
final AbstractRunningQuery runningQuery2 = queryEngine.eval(UUID.randomUUID(),
query2, new ListBindingSet());
- runningQuery2.setDeadline(now + 20000);
+ final long deadline2Millis = now + 20000/* millis */;
+
+ runningQuery2.setDeadline(deadline2Millis);
final QueryDeadline queryDeadline1 = new QueryDeadline(
- runningQuery1.getDeadline(), runningQuery1);
+ TimeUnit.MILLISECONDS.toNanos(runningQuery1.getDeadline()),
+ runningQuery1);
final QueryDeadline queryDeadline2 = new QueryDeadline(
- runningQuery2.getDeadline(), runningQuery2);
+ TimeUnit.MILLISECONDS.toNanos(runningQuery2.getDeadline()),
+ runningQuery2);
// The earlier deadline is LT the later deadline.
assertTrue(queryDeadline1.compareTo(queryDeadline2) < 0);
@@ -180,6 +187,15 @@
assertEquals(0, queryDeadline1.compareTo(queryDeadline1));
assertEquals(0, queryDeadline2.compareTo(queryDeadline2));
+ /*
+ * Verify that the query deadline (millis) was converted to nanos for
+ * QueryDeadline object.
+ */
+ assertEquals(TimeUnit.MILLISECONDS.toNanos(deadline1Millis),
+ queryDeadline1.deadlineNanos);
+ assertEquals(TimeUnit.MILLISECONDS.toNanos(deadline2Millis),
+ queryDeadline2.deadlineNanos);
+
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -87,7 +87,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @see TestFederatedQueryEngine
*/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-14 22:21:47
|
Revision: 8548
http://sourceforge.net/p/bigdata/code/8548
Author: tobycraig
Date: 2014-07-14 22:21:43 +0000 (Mon, 14 Jul 2014)
Log Message:
-----------
Fixed status tab links
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-14 20:22:02 UTC (rev 8547)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-14 22:21:43 UTC (rev 8548)
@@ -1367,8 +1367,8 @@
function getStatusNumbers(data) {
$('#status-text').html(data);
- $('#status-text a').eq(-2).click(function(e) { e.preventDefault(); showQueries(false); return false; });
- $('#status-text a').eq(-1).click(function(e) { e.preventDefault(); showQueries(true); return false; });
+ $('p:contains(Show queries, query details)').find('a').eq(0).click(function(e) { e.preventDefault(); showQueries(false); });
+ $('p:contains(Show queries, query details)').find('a').eq(1).click(function(e) { e.preventDefault(); showQueries(true); });
}
$('#show-queries').click(function(e) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-14 20:22:06
|
Revision: 8547
http://sourceforge.net/p/bigdata/code/8547
Author: thompsonbry
Date: 2014-07-14 20:22:02 +0000 (Mon, 14 Jul 2014)
Log Message:
-----------
Removed old deadline field and replace it in toString() with the runState.getDeadline(). This was missed when adding support for robust deadline termination for queries.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-14 20:08:31 UTC (rev 8546)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-14 20:22:02 UTC (rev 8547)
@@ -146,13 +146,13 @@
/** The unique identifier for this query. */
final private UUID queryId;
- /**
- * The query deadline. The value is the system clock time in milliseconds
- * when the query is due and {@link Long#MAX_VALUE} if there is no deadline.
- * In order to have a guarantee of a consistent clock, the deadline is
- * interpreted by the query controller.
- */
- final private AtomicLong deadline = new AtomicLong(Long.MAX_VALUE);
+// /**
+// * The query deadline. The value is the system clock time in milliseconds
+// * when the query is due and {@link Long#MAX_VALUE} if there is no deadline.
+// * In order to have a guarantee of a consistent clock, the deadline is
+// * interpreted by the query controller.
+// */
+// final private AtomicLong deadline = new AtomicLong(Long.MAX_VALUE);
/**
* The timestamp (ms) when the query begins to execute.
@@ -1744,7 +1744,7 @@
lock.lock();
try {
sb.append(",elapsed=" + getElapsed());
- sb.append(",deadline=" + deadline.get());
+ sb.append(",deadline=" + runState.getDeadline());
sb.append(",isDone=" + isDone());
sb.append(",isCancelled=" + isCancelled());
sb.append(",runState=" + runState);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-14 20:08:33
|
Revision: 8546
http://sourceforge.net/p/bigdata/code/8546
Author: tobycraig
Date: 2014-07-14 20:08:31 +0000 (Mon, 14 Jul 2014)
Log Message:
-----------
Added execution time to query history
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-07-14 13:38:43 UTC (rev 8545)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-07-14 20:08:31 UTC (rev 8546)
@@ -142,6 +142,7 @@
<th>Time</th>
<th>Query</th>
<th>Results</th>
+ <th>Execution Time</th>
<th>Delete</th>
</tr>
</thead>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-14 13:38:43 UTC (rev 8545)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-14 20:08:31 UTC (rev 8546)
@@ -720,6 +720,7 @@
// clear the old results and set the time to now
$(row).find('.query-time').text(new Date().toISOString());
$(row).find('.query-results').text('...');
+ $(row).find('.query-execution-time').text('...');
// move it to the top
$(row).prependTo('#query-history tbody');
queryExists = true;
@@ -736,6 +737,7 @@
a.text(query);
a.html(a.html().replace(/\n/g, '<br>'));
row.append('<td class="query-results">...</td>');
+ row.append('<td class="query-execution-time">...</td>');
row.append('<td class="query-delete"><a href="#">X</a></td>')
}
@@ -900,8 +902,27 @@
$('#download-link').remove();
}
-function updateResultCount(count) {
+function updateresultCountAndExecutionTime(count) {
$('#query-history tbody tr:first td.query-results').text(count);
+
+ var ms = Date.now() - Date.parse($('#query-history tbody tr:first td.query-time').html());
+ var sec = Math.floor(ms / 1000);
+ ms = ms % 1000;
+ var min = Math.floor(sec / 60);
+ min = min % 60;
+ var hr = Math.floor(min / 60);
+ var executionTime = '';
+ if(hr > 0) {
+ executionTime += hr + 'hr, ';
+ }
+ if(min > 0) {
+ executionTime += min + 'min, ';
+ }
+ if(sec > 0) {
+ executionTime += sec + 'sec, ';
+ }
+ executionTime += ms + 'ms';
+ $('#query-history tbody tr:first td.query-execution-time').html(executionTime);
}
function showQueryResults(data) {
@@ -933,7 +954,7 @@
table.append(tr);
}
}
- updateResultCount(rows.length);
+ updateresultCountAndExecutionTime(rows.length);
} else {
// JSON
// save data for export and pagination
@@ -942,7 +963,7 @@
if(typeof(data.boolean) != 'undefined') {
// ASK query
table.append('<tr><td>' + data.boolean + '</td></tr>').addClass('boolean');
- updateResultCount('' + data.boolean);
+ updateresultCountAndExecutionTime('' + data.boolean);
return;
}
@@ -984,7 +1005,7 @@
table.append(thead);
$('#total-results').html(data.results.bindings.length);
- updateResultCount(data.results.bindings.length);
+ updateresultCountAndExecutionTime(data.results.bindings.length);
setNumberOfPages();
showPage(1);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-07-14 13:38:53
|
Revision: 8545
http://sourceforge.net/p/bigdata/code/8545
Author: martyncutcher
Date: 2014-07-14 13:38:43 +0000 (Mon, 14 Jul 2014)
Log Message:
-----------
Reset the FixedAllocator.m_freeWaiting state field when moving between ContextAllocation and main RWStore free lists to maintain invariants
Modified Paths:
--------------
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-07-14 12:20:10 UTC (rev 8544)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-07-14 13:38:43 UTC (rev 8545)
@@ -50,11 +50,6 @@
private static final Logger log = Logger.getLogger(FixedAllocator.class);
- // private static final int cSmallSlotThreshold = 4096; // over half allocatable bits
- private static final int cSmallSlotThreshold = 4096; // debug test
-
- private static final int cSmallSlot = 1024; // debug test
-
private final int cModAllocation = 1 << RWStore.ALLOCATION_SCALEUP;
private final int cMinAllocation = cModAllocation * 1; // must be multiple of cModAllocation
@@ -209,7 +204,10 @@
private ArrayList m_freeList;
public void setFreeList(final ArrayList list) {
- m_freeList = list;
+ if (m_freeList != list) {
+ m_freeList = list;
+ m_freeWaiting = true;
+ }
if (!m_pendingContextCommit && hasFree() && meetsSmallSlotThreshold()) {
addToFreeList();
@@ -847,8 +845,8 @@
}
// then check for small slots
- if (m_size < cSmallSlot) { // invalid
- return m_freeBits > cSmallSlotThreshold;
+ if (m_size <= m_store.cSmallSlot) { // it's a small slotSMALL_SLOT_TYPE
+ return m_freeBits > m_store.cSmallSlotThreshold;
} else {
return true;
}
@@ -891,7 +889,7 @@
int addr = -1;
// Special allocation for small slots
- if (m_size <= cSmallSlot) {
+ if (m_size <= m_store.cSmallSlot) {
return allocFromIndex(size);
}
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-07-14 12:20:10 UTC (rev 8544)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-07-14 13:38:43 UTC (rev 8545)
@@ -357,6 +357,24 @@
String DEFAULT_FREE_BITS_THRESHOLD = "300";
/**
+ * Defines the size of a slot that defines it as a small slot.
+ * <p>
+ * Any slot equal to or less than this is considered a small slot and
+ * its availability for allocation is restricted to ensure a high
+ * chance that contiguous allocations can be made.
+ * <p>
+ * This is arranged by only returning small slot allocators to the free list
+ * if they have greater than 50% available slots, and then only allocating
+ * slots from sparse regions with >= 50% free/committed bits.
+ * <p>
+ * Small slot processing can be disabled by setting the smallSlotType to zero.
+ */
+ String SMALL_SLOT_TYPE = RWStore.class.getName() + ".smallSlotType";
+
+ // String DEFAULT_SMALL_SLOT_TYPE = "1024"; // standard default
+ String DEFAULT_SMALL_SLOT_TYPE = "0"; // initial default to no special processing
+
+ /**
* When <code>true</code>, scattered writes which are strictly ascending
* will be coalesced within a buffer and written out as a single IO
* (default {@value #DEFAULT_DOUBLE_BUFFER_WRITES}). This improves write
@@ -820,7 +838,16 @@
throw new IllegalArgumentException(Options.FREE_BITS_THRESHOLD
+ " : Must be between 1 and 5000");
}
-
+
+ cSmallSlot = Integer.valueOf(fileMetadata.getProperty(
+ Options.SMALL_SLOT_TYPE,
+ Options.DEFAULT_SMALL_SLOT_TYPE));
+
+ if (cSmallSlot < 0 || cSmallSlot > 2048) {
+ throw new IllegalArgumentException(Options.SMALL_SLOT_TYPE
+ + " : Must be between 0 and 2048");
+ }
+
m_metaBits = new int[m_metaBitsSize];
m_metaTransientBits = new int[m_metaBitsSize];
@@ -3631,6 +3658,10 @@
*/
final int cDefaultFreeBitsThreshold;
+ final int cSmallSlotThreshold = 4096; // debug test
+
+ int cSmallSlot = 1024; // debug test
+
/**
* Each "metaBit" is a file region
*/
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-07-14 12:20:10 UTC (rev 8544)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-07-14 13:38:43 UTC (rev 8545)
@@ -639,6 +639,26 @@
}
+ protected IRawStore getSmallSlotStore() {
+
+ return getSmallSlotStore(0);
+
+ }
+
+ protected IRawStore getSmallSlotStore(final int slotSize) {
+
+ final Properties properties = new Properties(getProperties());
+
+ properties.setProperty(
+ AbstractTransactionService.Options.MIN_RELEASE_AGE, "0");
+
+ properties.setProperty(
+ RWStore.Options.SMALL_SLOT_TYPE, "" + slotSize);
+
+ return getStore(properties);
+
+ }
+
protected Journal getStore(final long retentionMillis) {
final Properties properties = new Properties(getProperties());
@@ -881,7 +901,7 @@
*/
public void test_smallSlotRecycling() {
- final Journal store = (Journal) getStore();
+ final Journal store = (Journal) getSmallSlotStore(1024);
try {
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties 2014-07-14 12:20:10 UTC (rev 8544)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties 2014-07-14 13:38:43 UTC (rev 8545)
@@ -10,6 +10,8 @@
# Disk is the worm store. DiskRW is the read/write store.
com.bigdata.journal.AbstractJournal.bufferMode=DiskRW
+com.bigdata.rwstore.RWStore.smallSlotType=1024
+
com.bigdata.btree.writeRetentionQueue.capacity=8000
com.bigdata.btree.BTree.branchingFactor=512
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-07-14 12:20:14
|
Revision: 8544
http://sourceforge.net/p/bigdata/code/8544
Author: martyncutcher
Date: 2014-07-14 12:20:10 +0000 (Mon, 14 Jul 2014)
Log Message:
-----------
remove DEBUG rwstore logging level
Modified Paths:
--------------
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties 2014-07-13 14:54:11 UTC (rev 8543)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties 2014-07-14 12:20:10 UTC (rev 8544)
@@ -7,7 +7,7 @@
log4j.logger.com.bigdata=WARN
log4j.logger.com.bigdata.btree=WARN
-log4j.logger.com.bigdata.rwstore=DEBUG
+#log4j.logger.com.bigdata.rwstore=DEBUG
log4j.logger.com.bigdata.counters.History=ERROR
log4j.logger.com.bigdata.counters.XMLUtility$MyHandler=ERROR
log4j.logger.com.bigdata.counters.query.CounterSetQuery=INFO
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-07-13 14:54:22
|
Revision: 8543
http://sourceforge.net/p/bigdata/code/8543
Author: martyncutcher
Date: 2014-07-13 14:54:11 +0000 (Sun, 13 Jul 2014)
Log Message:
-----------
Initial commit to branch with LUBM test support
Modified Paths:
--------------
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/RWStore.properties
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.properties
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.xml
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsNoHistory.java
Added Paths:
-----------
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.properties
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.xml
branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/results.txt
Property Changed:
----------------
branches/BIGDATA_MGC_SMALL_SLOTS/
Index: branches/BIGDATA_MGC_SMALL_SLOTS
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS 2014-07-13 14:54:11 UTC (rev 8543)
Property changes on: branches/BIGDATA_MGC_SMALL_SLOTS
___________________________________________________________________
Modified: svn:ignore
## -29,3 +29,4 ##
bsbm10-dataset.nt.gz
bsbm10-dataset.nt.zip
benchmark*
+notes.txt
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-07-13 14:54:11 UTC (rev 8543)
@@ -49,6 +49,11 @@
public class FixedAllocator implements Allocator {
private static final Logger log = Logger.getLogger(FixedAllocator.class);
+
+ // private static final int cSmallSlotThreshold = 4096; // over half allocatable bits
+ private static final int cSmallSlotThreshold = 4096; // debug test
+
+ private static final int cSmallSlot = 1024; // debug test
private final int cModAllocation = 1 << RWStore.ALLOCATION_SCALEUP;
private final int cMinAllocation = cModAllocation * 1; // must be multiple of cModAllocation
@@ -206,18 +211,21 @@
public void setFreeList(final ArrayList list) {
m_freeList = list;
- if (!m_pendingContextCommit && hasFree()) {
- m_freeList.add(this);
- m_freeWaiting = false;
+ if (!m_pendingContextCommit && hasFree() && meetsSmallSlotThreshold()) {
+ addToFreeList();
}
+
}
/**
* To support postHACommit an allocator can be removed from the current freelist
*/
void removeFromFreeList() {
- if (m_freeList != null)
+ if (m_freeList != null) {
+ // log.warn("Removing allocator " + m_index + " from free list");
m_freeList.remove(this);
+ m_freeWaiting = true;
+ }
}
@@ -405,9 +413,13 @@
final int calcFree = calcFreeBits();
final int calcLiveFree = calcLiveFreeBits();
- return m_freeBits == calcFree
+ final boolean ret = m_freeBits == calcFree
&& (m_freeBits + m_freeTransients) == calcLiveFree;
+
+ if (!ret)
+ throw new AssertionError("m_free: " + m_freeBits + ", calcFree: " + calcFree);
+ return ret;
}
// read does not read in m_size since this is read to determine the class of
@@ -466,6 +478,13 @@
private int m_startAddr = 0;
private int m_endAddr = 0;
+
+ /**
+ * For "small slot" allocators the allocation search is
+ * always from bit areas with less than a maximum density to
+ * ensure that writes have better locality.
+ */
+ int m_allocIndex = -1;
/**
* The #of int32 values in a single {@link AllocBlock} region. The
@@ -533,6 +552,51 @@
}
/**
+ * find the allocationIndex of first "sparsely committed" AllocBlock.
+ *
+ * Checks the committed bits of all the AllocBlocks until one is found with
+ * > 50% free (or less than 50% allocated) of the committed bits.
+ * @param store
+ * @param i
+ */
+ void resetAllocIndex() {
+ resetAllocIndex(0);
+ }
+
+ void resetAllocIndex(final int start) {
+ m_allocIndex = start;
+
+ if (m_size <= 1024) {
+ for (int a = m_allocIndex/m_bitSize; a < m_allocBlocks.size(); a++) {
+ final AllocBlock ab = m_allocBlocks.get(a);
+
+ checkBlock(ab);
+
+ for (int i = (m_allocIndex%m_bitSize); i < m_bitSize; i++) {
+ // first check if transients are already full
+ if (ab.m_transients[i] != 0xFFFFFFFF) {
+ // then check maximum 50% commit allocated
+ if (Integer.bitCount(ab.m_commit[i]) < 16) {
+ final AllocBlock abr = m_allocBlocks.get(m_allocIndex/m_bitSize);
+ assert abr == ab;
+
+ return;
+ }
+ }
+ m_allocIndex++;
+ }
+ }
+
+ // must remove from free list if we cannot set the alloc Index for a small slot
+ if (start == 0) {
+ removeFromFreeList();
+ } else {
+ resetAllocIndex(0);
+ }
+ }
+ }
+
+ /**
* This determines the size of the reservation required in terms of
* the number of ints each holding bits for 32 slots.
*
@@ -693,6 +757,7 @@
try {
if (log.isDebugEnabled())
checkBits();
+
if (((AllocBlock) m_allocBlocks.get(block))
.freeBit(offset % nbits, m_sessionActive && !overideSession)) { // bit adjust
@@ -721,7 +786,7 @@
if (log.isDebugEnabled())
checkBits();
- return true;
+ return true;
} else if (addr >= m_startAddr && addr < m_endAddr) {
final Iterator<AllocBlock> iter = m_allocBlocks.iterator();
@@ -755,112 +820,227 @@
private void checkFreeList() {
if (m_freeWaiting && !m_pendingContextCommit) {
- if (m_freeBits >= m_store.cDefaultFreeBitsThreshold) {
- m_freeWaiting = false;
+ if (meetsSmallSlotThreshold()) {
- if (log.isDebugEnabled())
- log.debug("Returning Allocator to FreeList - " + m_size);
+ addToFreeList();
- m_freeList.add(this);
+ resetAllocIndex(0);
}
}
}
+
+ private void addToFreeList() {
+ assert m_freeWaiting;
+
+ m_freeWaiting = false;
+ m_freeList.add(this);
+ m_allocIndex = -1;
+
+ if (log.isDebugEnabled())
+ log.debug("Returning Allocator to FreeList - " + m_size);
+ }
+
+ private boolean meetsSmallSlotThreshold() {
+ // check threshold for all slots
+ if (m_freeBits < m_store.cDefaultFreeBitsThreshold) {
+ return false;
+ }
+
+ // then check for small slots
+ if (m_size < cSmallSlot) { // invalid
+ return m_freeBits > cSmallSlotThreshold;
+ } else {
+ return true;
+ }
+ }
/**
* The introduction of IAllocationContexts has added some complexity to
* the older concept of a free list. With AllocationContexts it is
* possibly for allocator to have free space available but this being
- * restricted to a specific AllocaitonContext. The RWStore alloc method
- * must therefore handle the
+ * restricted to a specific AllocationContext.
+ * <p>
+ * In addition to the standard free allocation search we want to add a
+ * "density" restriction for small slots to encourage the aggregation
+ * of writes (by increasing the likelihood of sibling slot allocation).
+ * <p>
+ * There is some "Do What I mean" complexity here, with difficulty in
+ * determining a good rule to identify an initial allocation point. There
+ * is a danger of significantly reducing the allocation efficiency of
+ * short transactions if we too naively check committed bit density. We
+ * should only do this when identifying the initial allocation, and when
+ * the allocIndex is incremented.
*/
- public int alloc(final RWStore store, final int size, final IAllocationContext context) {
+ public int alloc(final RWStore store, final int size,
+ final IAllocationContext context) {
try {
- if (size <= 0)
- throw new IllegalArgumentException(
- "Allocate requires positive size, got: " + size);
+ if (size <= 0)
+ throw new IllegalArgumentException(
+ "Allocate requires positive size, got: " + size);
- if (size > m_size)
- throw new IllegalArgumentException(
- "FixedAllocator with slots of " + m_size
- + " bytes requested allocation for "+ size + " bytes");
+ if (size > m_size)
+ throw new IllegalArgumentException(
+ "FixedAllocator with slots of " + m_size
+ + " bytes requested allocation for " + size
+ + " bytes");
- int addr = -1;
+ if (m_freeBits == 0) {
+ throw new IllegalStateException("Request to allocate from " + m_size + "byte slot FixedAllocator with zero bits free - should not be on the Free List");
+ }
+
+ int addr = -1;
+
+ // Special allocation for small slots
+ if (m_size <= cSmallSlot) {
+ return allocFromIndex(size);
+ }
- final Iterator<AllocBlock> iter = m_allocBlocks.iterator();
- int count = -1;
- while (addr == -1 && iter.hasNext()) {
- count++;
+ final Iterator<AllocBlock> iter = m_allocBlocks.iterator();
+ int count = -1;
+ while (addr == -1 && iter.hasNext()) {
+ count++;
- final AllocBlock block = iter.next();
- if (block.m_addr == 0) {
- int blockSize = 32 * m_bitSize;
+ final AllocBlock block = iter.next();
+ checkBlock(block);
+
+ addr = block.alloc(m_size);
+ }
+
+ if (addr != -1) {
+
+ addr += 3; // Tweak to ensure non-zero address for offset 0
+
+ if (--m_freeBits == 0) {
+ if (log.isTraceEnabled())
+ log.trace("Remove from free list");
+ removeFromFreeList();
+
+ // Should have been first on list, now check for first
+ if (m_freeList.size() > 0) {
+ if (log.isDebugEnabled()) {
+ final FixedAllocator nxt = (FixedAllocator) m_freeList
+ .get(0);
+ log.debug("Freelist head: " + nxt.getSummaryStats());
+ }
+ }
+ }
+
+ addr += (count * 32 * m_bitSize);
+
+ final int value = -((m_index << RWStore.OFFSET_BITS) + addr);
+
if (m_statsBucket != null) {
- m_statsBucket.addSlots(blockSize);
+ m_statsBucket.allocate(size);
}
- blockSize *= m_size;
- blockSize >>= RWStore.ALLOCATION_SCALEUP;
- block.m_addr = grabAllocation(store, blockSize);
- if (log.isDebugEnabled())
- log.debug("Allocation block at " + block.m_addr + " of " + (blockSize << 16) + " bytes");
+ return value;
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append("FixedAllocator returning null address, with freeBits: "
+ + m_freeBits + "\n");
- if (m_startAddr == 0) {
- m_startAddr = block.m_addr;
+ for (AllocBlock ab : m_allocBlocks) {
+ sb.append(ab.show() + "\n");
}
- m_endAddr = block.m_addr - blockSize;
+
+ log.error(sb);
+
+ return 0;
}
- addr = block.alloc(m_size);
+ } finally {
+ if (log.isDebugEnabled())
+ checkBits();
}
+ }
+
+ void checkBlock(final AllocBlock block) {
+ if (block.m_addr == 0) {
+ int blockSize = 32 * m_bitSize;
+ if (m_statsBucket != null) {
+ m_statsBucket.addSlots(blockSize);
+ }
+ blockSize *= m_size;
+ blockSize >>= RWStore.ALLOCATION_SCALEUP;
- if (addr != -1) {
+ block.m_addr = grabAllocation(m_store, blockSize);
+ if (log.isDebugEnabled())
+ log.debug("Allocation block at " + block.m_addr
+ + " of " + (blockSize << 16) + " bytes");
- addr += 3; // Tweak to ensure non-zero address for offset 0
-
- if (--m_freeBits == 0) {
- if (log.isTraceEnabled())
- log.trace("Remove from free list");
- m_freeList.remove(this);
- m_freeWaiting = true;
-
- // Should have been first on list, now check for first
- if (m_freeList.size() > 0) {
- if (log.isDebugEnabled()) {
- final FixedAllocator nxt = (FixedAllocator) m_freeList.get(0);
- log.debug("Freelist head: " + nxt.getSummaryStats());
- }
- }
+ if (m_startAddr == 0) {
+ m_startAddr = block.m_addr;
}
+ m_endAddr = block.m_addr - blockSize;
+ }
- addr += (count * 32 * m_bitSize);
-
- final int value = -((m_index << RWStore.OFFSET_BITS) + addr);
+ }
+
+ int allocFromIndex(final int size) {
+
+ if (m_allocIndex == -1) {
+ resetAllocIndex();
- if (m_statsBucket != null) {
- m_statsBucket.allocate(size);
+ if (m_allocIndex == -1) {
+ throw new AssertionError("Unable to set AllocIndex with m_freeBits: " + m_freeBits);
}
-
-
- return value;
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append("FixedAllocator returning null address, with freeBits: " + m_freeBits + "\n");
+ }
- for (AllocBlock ab: m_allocBlocks) {
- sb.append(ab.show() + "\n");
- }
-
- log.error(sb);
+ if (log.isDebugEnabled())
+ checkBits();
- return 0;
+
+ if (m_freeBits != calcFreeBits()) {
+ final int calc = calcFreeBits();
+ throw new AssertionError("m_freeBits != calcFreeBits() : " + m_freeBits + "!=" + calc);
}
- } finally {
- if (log.isDebugEnabled())
- checkBits();
+
+ // there MUST be bits free in the m_allocIndex block
+ final AllocBlock ab = m_allocBlocks.get(m_allocIndex/m_bitSize);
+
+ if (ab.m_addr == 0) {
+ throw new AssertionError("No allocation for AllocBlock with m_allocIndex: " + m_allocIndex);
}
+
+ final int abblock = m_allocIndex % m_bitSize;
+
+ assert ab.m_transients[abblock] != 0xFFFFFFFF; // not all set
+
+ final int bit = RWStore.fndBit(ab.m_transients[abblock]);
+
+ assert bit >= 0;
+
+ m_freeBits--;
+
+ final int abit = (abblock*32) + bit;
+ RWStore.setBit(ab.m_live, abit);
+ RWStore.setBit(ab.m_transients, abit);
+
+ // Note +3 for address teak for special low order bits
+ final int addr = -((m_index << RWStore.OFFSET_BITS) + (m_allocIndex*32) + (bit + 3));
+
+ // Now check current index
+ if (ab.m_transients[abblock] == 0xFFFFFFFF) {
+ // find next allocIndex
+ resetAllocIndex(m_allocIndex+1);
+ }
+
+ if (m_freeBits != calcFreeBits()) {
+ throw new AssertionError("m_freeBits != calcFreeBits()");
+ }
+ // assert m_freeBits == calcFreeBits();
+
+ if (m_statsBucket != null) {
+ m_statsBucket.allocate(size);
+ }
+
+ return addr;
}
protected int grabAllocation(RWStore store, int blockSize) {
- return store.allocBlock(blockSize);
+
+ final int ret = store.allocBlock(blockSize);
+
+ return ret;
}
public boolean hasFree() {
@@ -1040,9 +1220,16 @@
}
m_freeTransients = transientbits();
+ m_freeBits = calcFreeBits();
+ // Ensure allocIndex is reset
+ m_allocIndex = -1;
+
assert calcSessionFrees();
+ if (log.isDebugEnabled())
+ checkBits();
+
return isolatedWrites;
}
@@ -1145,6 +1332,8 @@
}
ab.setBitExternal(bit);
+
+ m_freeBits--;
}
public int getSlotSize() {
@@ -1231,8 +1420,8 @@
if (m_pendingContextCommit) {
m_pendingContextCommit = false;
- if (hasFree()) {
- m_freeList.add(this);
+ if (m_freeWaiting && meetsSmallSlotThreshold()) {
+ addToFreeList();
}
}
@@ -1241,15 +1430,17 @@
// Handle re-addition to free list once transient frees are
// added back
- if (m_freeWaiting && m_freeBits >= m_store.cDefaultFreeBitsThreshold) {
- m_freeList.add(this);
- m_freeWaiting = false;
+ if (m_freeWaiting && meetsSmallSlotThreshold()) {
+ addToFreeList();
}
m_freeTransients = 0;
}
+ if (log.isDebugEnabled())
+ checkBits();
+
}
/*
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-07-13 14:54:11 UTC (rev 8543)
@@ -2669,6 +2669,10 @@
}
final int addr = allocator.alloc(this, size, context);
+
+ if (addr == 0) {
+ throw new IllegalStateException("Free Allocator unable to allocate address: " + allocator.getSummaryStats());
+ }
if (allocator.isUnlocked() && !m_commitList.contains(allocator)) {
m_commitList.add(allocator);
@@ -2678,6 +2682,9 @@
final long pa = physicalAddress(addr);
if (pa == 0L) {
+ // aid to debug
+ physicalAddress(addr);
+
throw new IllegalStateException(
"No physical address found for " + addr);
}
@@ -3360,7 +3367,7 @@
+ m_metaBitsAddr + ", active contexts: "
+ m_contexts.size());
- if (log.isDebugEnabled() && m_quorum.isHighlyAvailable()) {
+ if (log.isDebugEnabled() && m_quorum != null && m_quorum.isHighlyAvailable()) {
log.debug(showAllocatorList());
@@ -3952,7 +3959,7 @@
private void extendFile() {
final int adjust = -1200 + (m_fileSize / 10);
-
+
extendFile(adjust);
}
@@ -4050,12 +4057,21 @@
static int fndBit(final int[] bits, final int offset, final int size) {
final int eob = size + offset;
- for (int i = offset; i < eob; i++) {
- if (bits[i] != 0xFFFFFFFF) {
- for (int k = 0; k < 32; k++) {
- if ((bits[i] & (1 << k)) == 0) {
- return (i * 32) + k;
- }
+ for (int i = offset; i < eob; i++) {
+ final int b = fndBit(bits[i]);
+ if (b != -1) {
+ return (i * 32) + b;
+ }
+ }
+
+ return -1;
+ }
+
+ static int fndBit(final int bits) {
+ if (bits != 0xFFFFFFFF) {
+ for (int k = 0; k < 32; k++) {
+ if ((bits & (1 << k)) == 0) {
+ return k;
}
}
}
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/resources/logging/log4j.properties 2014-07-13 14:54:11 UTC (rev 8543)
@@ -7,6 +7,7 @@
log4j.logger.com.bigdata=WARN
log4j.logger.com.bigdata.btree=WARN
+log4j.logger.com.bigdata.rwstore=DEBUG
log4j.logger.com.bigdata.counters.History=ERROR
log4j.logger.com.bigdata.counters.XMLUtility$MyHandler=ERROR
log4j.logger.com.bigdata.counters.query.CounterSetQuery=INFO
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-07-13 14:54:11 UTC (rev 8543)
@@ -830,6 +830,114 @@
}
/**
+ * Ensures the allocation of unique addresses by mapping allocated
+ * address with uniqueness assertion against physical address.
+ */
+ public void test_addressingContiguous() {
+
+ final Journal store = (Journal) getStore();
+
+ try {
+
+ final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();
+
+ final RWStore rw = bufferStrategy.getStore();
+ final int cSlotSize = 128;
+ final int cAllocSize = 99;
+
+ long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
+ for (int i = 0; i < 500000; i++) {
+ final int a = rw.alloc(cAllocSize, null);
+ final long pa = rw.physicalAddress(a);
+
+ if (pa != (pap+cSlotSize)) {
+ // for debug
+ rw.physicalAddress(a);
+ fail("Non-Contiguous slots: " + i + ", " + pa + "!=" + (pap+cSlotSize));
+ }
+
+ pap = pa;
+
+ }
+
+ store.commit();
+
+ final StringBuilder sb = new StringBuilder();
+ rw.showAllocators(sb);
+
+ log.warn(sb.toString());
+
+ } finally {
+
+ store.destroy();
+
+ }
+
+ }
+
+ /**
+ * Tests the recycling of small slot alloctors and outputs statistics related
+ * to contiguous allocations indicative of reduced IOPS.
+ */
+ public void test_smallSlotRecycling() {
+
+ final Journal store = (Journal) getStore();
+
+ try {
+
+ final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();
+
+ final RWStore rw = bufferStrategy.getStore();
+ final int cSlotSize = 128;
+ final int cAllocSize = 99;
+
+ int breaks = 0;
+ int contiguous = 0;
+
+ ArrayList<Integer> recycle = new ArrayList<Integer>();
+
+ long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
+ for (int i = 0; i < 500000; i++) {
+ final int a = rw.alloc(cSlotSize, null);
+ final long pa = rw.physicalAddress(a);
+
+ if (r.nextInt(7) < 5) { // more than 50% recycle
+ recycle.add(a);
+ }
+
+ if (pa == (pap+cSlotSize)) {
+ contiguous++;
+ } else {
+ breaks++;
+ }
+
+ pap = pa;
+
+ if (recycle.size() > 5000) {
+ log.warn("Transient Frees for immediate recyling");
+ for (int e : recycle) {
+ rw.free(e, cAllocSize);
+ }
+ recycle.clear();
+ }
+ }
+
+ store.commit();
+
+ final StringBuilder sb = new StringBuilder();
+ rw.showAllocators(sb);
+
+ log.warn("Contiguous: " + contiguous + ", breaks: " + breaks + "\n" + sb.toString());
+
+ } finally {
+
+ store.destroy();
+
+ }
+
+ }
+
+ /**
* Basic allocation test to ensure the FixedAllocators are operating
* efficiently.
*
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/RWStore.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/RWStore.properties 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/RWStore.properties 2014-07-13 14:54:11 UTC (rev 8543)
@@ -10,17 +10,44 @@
# Disk is the worm store. DiskRW is the read/write store.
com.bigdata.journal.AbstractJournal.bufferMode=DiskRW
-com.bigdata.btree.writeRetentionQueue.capacity=4000
+com.bigdata.btree.writeRetentionQueue.capacity=40000
com.bigdata.btree.BTree.branchingFactor=128
# Override branching factors for BSBM 100M to target 8k pages.
-com.bigdata.namespace.BSBM_284826.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=800
-com.bigdata.namespace.BSBM_284826.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=800
-com.bigdata.namespace.BSBM_284826.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=400
-com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.branchingFactor=1024
-com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=1024
-com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.BSBM_27850.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.BSBM_27850.spo.TERM2ID.com.bigdata.btree.BTree.writeRetentionQueue.capacity=10000
+#com.bigdata.namespace.BSBM_27850.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.BSBM_27850.spo.ID2TERM.com.bigdata.btree.BTree.writeRetentionQueue.capacity=10000
+#com.bigdata.namespace.BSBM_27850.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=400
+#com.bigdata.namespace.BSBM_27850.spo.SPO.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.SPO.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+# Override branching factors for BSBM 100M to target 8k pages.
+#com.bigdata.namespace.BSBM_284826.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=800
+#com.bigdata.namespace.BSBM_284826.spo.TERM2ID.com.bigdata.btree.BTree.writeRetentionQueue.capacity=6000
+#com.bigdata.namespace.BSBM_284826.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=800
+#com.bigdata.namespace.BSBM_284826.spo.ID2TERM.com.bigdata.btree.BTree.writeRetentionQueue.capacity=6000
+# com.bigdata.namespace.BSBM_284826.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=400
+# com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+# com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+# com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+
+#com.bigdata.namespace.BSBM_284826.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=512
+
# Override branching factors for BSBM 200M to target 8k pages.
com.bigdata.namespace.BSBM_566496.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=800
com.bigdata.namespace.BSBM_566496.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=800
@@ -30,13 +57,15 @@
com.bigdata.namespace.BSBM_566496.spo.OSP.com.bigdata.btree.BTree.branchingFactor=1024
# Override the #of write cache buffers.
-com.bigdata.journal.AbstractJournal.writeCacheBufferCount=12
+# com.bigdata.journal.AbstractJournal.writeCacheBufferCount=12
+com.bigdata.journal.AbstractJournal.writeCacheBufferCount=800
# Note: You must override the buffer capacity in build.xml on the
# "run-load" target, but this would give you 10M write cache buffers
# if you placed that override there.
#
# -Dcom.bigdata.io.DirectBufferPool.bufferCapacity=10485760
+-Dcom.bigdata.io.DirectBufferPool.bufferCapacity=1048576
# 200M initial extent.
com.bigdata.journal.AbstractJournal.initialExtent=209715200
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.properties 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.properties 2014-07-13 14:54:11 UTC (rev 8543)
@@ -3,7 +3,7 @@
# the base directory
bsbm.dir=.
# the bsbm distribution directory
-bsbmtools.dir=./bsbmtools
+bsbmtools.dir=/bsbmtools/trunk
# the bigdata base directory
bigdata.dir=../..
@@ -34,7 +34,7 @@
# use the CMS-I GC mode to minimize latency at the expense of some throughput.
# The port at which the NanoSparqlServer will respond (if started).
-bsbm.nanoServerPort=80
+bsbm.nanoServerPort=8080
# The maximum size of the java heap for the BSBM test runs.
bsbm.maxMem=4g
@@ -44,6 +44,7 @@
#bsbm.pc=10
#bsbm.pc=2785
#bsbm.pc=70812
+#bsbm.pc=27850
bsbm.pc=284826
#bsbm.pc=566496
@@ -51,7 +52,8 @@
bsbm.namespace=BSBM_${bsbm.pc}
# The data directory for the generated benchmark files.
-bsbm.baseDir=${bsbmtools.dir}/td_100m
+#bsbm.baseDir=${bsbmtools.dir}/td_100m
+bsbm.baseDir=${bsbmtools.dir}/bsbm_${bsbm.pc}/
# Laptop
#bsbm.baseDir=d:/bigdata-perf-analysis/bsbm3/bsbm_${bsbm.pc}
# Server
@@ -68,8 +70,8 @@
bsbm.outputType=nt
# Specify ".gz" or ".zip" if pre-generated files have been compressed.
-#bsbm.compressType=
-bsbm.compressType=".gz"
+bsbm.compressType=
+#bsbm.compressType=".gz"
# Which mode to use for the Journal. (DiskRW or DiskWORM)
journalMode=RW
@@ -84,7 +86,8 @@
# The name of the file used for the BSBM journal.
#bsbm.journalFile=${bsbm.baseDir}/bsbm_${bsbm.pc}/bigdata-bsbm.${journalMode}.jnl
# Note: This is on the large volume.
-bsbm.journalFile=/data/bsbm/bsbm3_${bsbm.pc}/bigdata-bsbm.${journalMode}.jnl
+#bsbm.journalFile=/Volumes/NonSSD/bsbm/bsbm3_${bsbm.pc}/bigdata-bsbm.${journalMode}.jnl
+bsbm.journalFile=/bsbm/bsbm3_${bsbm.pc}/bigdata-bsbm.${journalMode}.jnl
# Windows 2008 Server: SSD.
#bsbm.journalFile=e:/data/bsbm/bsbm_${bsbm.pc}/bigdata-bsbm.${journalMode}.jnl
# Windows 2008 Server: SAS.
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.xml
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.xml 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/bsbm3/build.xml 2014-07-13 14:54:11 UTC (rev 8543)
@@ -24,8 +24,9 @@
<arg line="-namespace ${bsbm.namespace} ${bsbm.journalPropertyFile} ${bsbm.outputFile}.${bsbm.outputType}${bsbm.compressType}" />
<!-- specify/override the journal file name. -->
<jvmarg line="${queryJvmArgs} -Dcom.bigdata.journal.AbstractJournal.file=${bsbm.journalFile}
- -Dcom.bigdata.rdf.store.DataLoader.bufferCapacity=1000000
- -Dcom.bigdata.io.DirectBufferPool.bufferCapacity=10485760
+ -Dcom.bigdata.rdf.store.DataLoader.bufferCapacity=1000000
+ -Dcom.bigdata.rdf.store.DataLoader.commit=Incremental
+ -Dcom.bigdata.io.DirectBufferPool.bufferCapacity=1048576
" />
<classpath>
<path refid="runtime.classpath" />
Added: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties (rev 0)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/RWStore.properties 2014-07-13 14:54:11 UTC (rev 8543)
@@ -0,0 +1,98 @@
+#
+# Note: These options are applied when the journal and the triple store are
+# created. If you want to modify options after than you must do so using
+# [ant set-properties] or by overriding appropriate properties on the command
+# line.
+
+##
+## Journal options.
+##
+
+# Disk is the worm store. DiskRW is the read/write store.
+com.bigdata.journal.AbstractJournal.bufferMode=DiskRW
+com.bigdata.btree.writeRetentionQueue.capacity=8000
+com.bigdata.btree.BTree.branchingFactor=512
+
+# Override branching factors for BSBM 100M to target 8k pages.
+#com.bigdata.namespace.BSBM_27850.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.BSBM_27850.spo.TERM2ID.com.bigdata.btree.BTree.writeRetentionQueue.capacity=10000
+#com.bigdata.namespace.BSBM_27850.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.BSBM_27850.spo.ID2TERM.com.bigdata.btree.BTree.writeRetentionQueue.capacity=10000
+#com.bigdata.namespace.BSBM_27850.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=400
+#com.bigdata.namespace.BSBM_27850.spo.SPO.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.SPO.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.POS.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.branchingFactor=64
+#com.bigdata.namespace.BSBM_27850.spo.OSP.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+
+# Override branching factors for BSBM 100M to target 8k pages.
+#com.bigdata.namespace.BSBM_284826.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=800
+#com.bigdata.namespace.BSBM_284826.spo.TERM2ID.com.bigdata.btree.BTree.writeRetentionQueue.capacity=6000
+#com.bigdata.namespace.BSBM_284826.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=800
+#com.bigdata.namespace.BSBM_284826.spo.ID2TERM.com.bigdata.btree.BTree.writeRetentionQueue.capacity=6000
+# com.bigdata.namespace.BSBM_284826.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=400
+# com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+# com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+# com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=64
+# com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.writeRetentionQueue.capacity=20000
+
+#com.bigdata.namespace.BSBM_284826.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.POS.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
+#com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=512
+
+# Override branching factors for BSBM 200M to target 8k pages.
+com.bigdata.namespace.LUBM.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=800
+com.bigdata.namespace.LUBM.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=800
+com.bigdata.namespace.LUBM.spo.SPO.com.bigdata.btree.BTree.branchingFactor=1024
+#com.bigdata.namespace.LUBM.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.LUBM.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=256
+#com.bigdata.namespace.LUBM.spo.SPO.com.bigdata.btree.BTree.branchingFactor=256
+
+# Override the #of write cache buffers.
+# com.bigdata.journal.AbstractJournal.writeCacheBufferCount=12
+com.bigdata.journal.AbstractJournal.writeCacheBufferCount=800
+
+# Note: You must override the buffer capacity in build.xml on the
+# "run-load" target, but this would give you 10M write cache buffers
+# if you placed that override there.
+#
+# -Dcom.bigdata.io.DirectBufferPool.bufferCapacity=10485760
+-Dcom.bigdata.io.DirectBufferPool.bufferCapacity=1048576
+
+# 200M initial extent.
+com.bigdata.journal.AbstractJournal.initialExtent=209715200
+com.bigdata.journal.AbstractJournal.maximumExtent=209715200
+
+##
+## Triple store options.
+##
+
+com.bigdata.rdf.store.AbstractTripleStore.quads=false
+com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false
+com.bigdata.rdf.store.AbstractTripleStore.textIndex=false
+# 50000 is the default.
+#com.bigdata.rdf.store.AbstractTripleStore.termCache.capacity=50000
+com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms
+
+# Tighter coding for the BSBM vocabulary.
+com.bigdata.rdf.store.AbstractTripleStore.vocabularyClass=com.bigdata.rdf.vocab.BSBMVocabulary
+
+# Inlining for "USD" datatype.
+com.bigdata.rdf.store.AbstractTripleStore.extensionFactoryClass=com.bigdata.rdf.internal.BSBMExtensionFactory
+
+##
+## Sail options.
+##
+
+com.bigdata.rdf.sail.truthMaintenance=false
+
+# 10000 is default.
+com.bigdata.rdf.sail.bufferCapacity=100000
Added: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.properties
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.properties (rev 0)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.properties 2014-07-13 14:54:11 UTC (rev 8543)
@@ -0,0 +1,135 @@
+# ant build properties.
+
+# the base directory
+lubm.dir=.
+# the lubm distribution directory
+lubmtools.dir=/lubmtools/trunk
+# the bigdata base directory
+bigdata.dir=../..
+
+# Where to find the pre-build bigdata classes.
+bigdata.build.dir=${bigdata.dir}/ant-build
+
+bigdata.install.lib.dir=${bigdata.dir}/
+
+##
+# javac options
+##
+
+# debug=on|off
+javac.debug=off
+# debuglevel=lines,vars,source (or any combination thereof).
+javac.debuglevel=lines,vars,source
+javac.verbose=off
+#javac.target=1.6
+#javac.source=1.6
+javac.encoding=Cp1252
+
+# lubm properties.
+#
+# Note: By default, the files will wind up in ./ant-build/bin
+#
+# Note: By degault, the server jvm will optimize for throughput and can have
+# high variation in throughput due to long GC pauses for larger heaps. You can
+# use the CMS-I GC mode to minimize latency at the expense of some throughput.
+
+# The port at which the NanoSparqlServer will respond (if started).
+lubm.nanoServerPort=8080
+
+# The maximum size of the java heap for the LUBM test runs.
+lubm.maxMem=4g
+
+# The #of products, which implies the #of triples. The relationship is linear.
+# E.g.: 10=5k; 2785=1M; 70812=25M; 284826=100M; 566496=200M; 1132992=400M, etc.
+lubm.pc=U800
+#lubm.pc=U50
+#lubm.pc=U100
+
+# The namespace of the KB instance (multiple KBs can be in the same database).
+lubm.namespace=LUBM
+
+lubm.outputType=owl
+
+# Specify ".gz" or ".zip" if pre-generated files have been compressed.
+lubm.compressType=
+
+
+# The data directory for the generated benchmark files.
+#lubm.baseDir=${lubmtools.dir}/td_100m
+#lubm.baseDir=/Volumes/SSDData/bigdata-perf/data/lubm/${lubm.pc}/data
+lubm.baseDir=/Volumes/NonSSD/lubm/${lubm.pc}/data
+
+# Which mode to use for the Journal. (DiskRW or DiskWORM)
+journalMode=RW
+#journalMode=WORM
+
+# The name of the file containing the generated RDF data without the filename extension.
+lubm.outputFile=${lubm.baseDir}/University0_0
+
+# The name of the file used to configure the LUBM journal.
+lubm.journalPropertyFile=${journalMode}Store.properties
+
+# The name of the file used for the BSBM journal.
+#lubm.journalFile=${lubm.baseDir}/lubm_${lubm.pc}/bigdata-lubm.${journalMode}.jnl
+# Note: This is on the large volume.
+#lubm.journalFile=/Volumes/NonSSD/lubm/lubm3_${lubm.pc}/bigdata-lubm.${journalMode}.jnl
+
+# standard load
+lubm.journalFile=/Volumes/SSDData/bigdata-perf/data/lubm/${lubm.pc}/bigdata-lubm-256-1024.RW.jnl
+
+# Use this to override journal copied to NonSSD volume
+#lubm.journalFile=/Volumes/NonSSD/lubm/bigdata-lubm-small-64-256.RW.jnl
+
+
+# Windows 2008 Server: SSD.
+#lubm.journalFile=e:/data/lubm/lubm_${lubm.pc}/bigdata-lubm.${journalMode}.jnl
+# Windows 2008 Server: SAS.
+#lubm.journalFile=f:/data/lubm/lubm_${lubm.pc}/bigdata-lubm.${journalMode}.jnl
+
+#
+# Profiler parameters.
+#
+
+# No profiler.
+profilerAgent=
+# linux-64
+#profilerAgent=-agentpath:/usr/java/yjp-9.0.3/bin/linux-x86-64/libyjpagent.so
+# Windows
+#profilerAgent="-agentpath:C:/Program Files/YourKit Java Profiler 9.0.2/bin/win32/yjpagent.dll"
+# Windows Server 2008
+#profilerAgent="-agentpath:C:/Program Files (x86)/YourKit Java Profiler 9.0.4/bin/win64/yjpagent.dll"
+
+# No profiler.
+profilerAgentOptions=
+# all profiling initially disabled.
+#profilerAgentOptions=-agentlib:yjpagent=disableexceptiontelemetry,disablestacktelemetry
+
+profiler=${profilerAgent} ${profilerAgentOptions}
+
+# Configure GC.
+#gcopts=
+#gcopts=-verbose:gc
+#gcopts=-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode
+gcopts=-XX:+UseParallelOldGC
+#gcopts=-XX:+UnlockExperimentalVMOptions -XX:+UseG1GC
+
+# Generates detailed logging on the JVM GC behavior. The service will
+# start in the configured service directory, so the log file will be in
+# that directory as well. The service directory is typically on local
+# disk, so that is where you need to look for this file.
+gcdebug=
+#gcdebug=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:jvm_gc.log
+
+# The record cache (empty for the default cache).
+cache=
+#cache=-Dcom.bigdata.LRUNexus.enabled=false
+#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true
+#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000
+#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache
+#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler
+#
+## -Dcom.bigdata.LRUNexus.percentHeap=.1
+
+# all jvm args for query.
+queryJvmArgs=-server -Xmx${lubm.maxMem} -showversion ${gcopts} ${gcdebug} ${profiler} ${cache} -Dlog4j.configuration=file:log4j.properties
+# -Dlog4j.debug
Added: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.xml
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.xml (rev 0)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/build.xml 2014-07-13 14:54:11 UTC (rev 8543)
@@ -0,0 +1,62 @@
+<!-- $Id: build.xml 5779 2011-12-14 18:17:54Z thompsonbry $ -->
+<!-- -->
+<!-- do "ant bundle-jar" in the parent directory first. -->
+<!-- -->
+<project name="lubm" basedir=".">
+
+ <property file="build.properties" />
+
+ <path id="runtime.classpath">
+ <!-- The bigdata dependencies (for the nano-server). -->
+ <fileset dir="${bigdata.build.dir}/lib">
+ <include name="**/*.jar" />
+ </fileset>
+ </path>
+
+ <!-- Note: split data files and use RDFDataLoadMaster for scale-out. -->
+ <target name="run-load"
+ description="Load a data set.">
+ <!-- delete file if it exists so we load into a new journal. -->
+ <delete file="${lubm.journalFile}" />
+ <java classname="com.bigdata.rdf.store.DataLoader"
+ fork="true" failonerror="true"
+ >
+ <arg line="-namespace ${lubm.namespace} ${lubm.journalPropertyFile} ${lubm.outputFile}.${lubm.outputType}${lubm.compressType}" />
+ <!-- specify/override the journal file name. -->
+ <jvmarg line="${queryJvmArgs} -Dcom.bigdata.journal.AbstractJournal.file=${lubm.journalFile}
+ -Dcom.bigdata.rdf.store.DataLoader.bufferCapacity=1000000
+ -Dcom.bigdata.rdf.store.DataLoader.commit=Incremental
+ -Dcom.bigdata.io.DirectBufferPool.bufferCapacity=1048576
+ " />
+ <classpath>
+ <path refid="runtime.classpath" />
+ </classpath>
+ </java>
+ </target>
+
+ <target name="start-sparql-server"
+ description="Start a small http server fronting for a bigdata database instance.">
+ <java classname="com.bigdata.rdf.sail.webapp.NanoSparqlServer"
+ fork="true" failonerror="true"
+ >
+ <arg line="${lubm.nanoServerPort} ${lubm.namespace} ${lubm.journalPropertyFile}" />
+ <!-- specify/override the journal file name. -->
+ <jvmarg line="${queryJvmArgs} -Dcom.bigdata.journal.AbstractJournal.file=${lubm.journalFile}" />
+ <classpath>
+ <path refid="runtime.classpath" />
+ </classpath>
+ </java>
+ </target>
+
+ <target name="dump-journal"
+ description="Dump interesting facts about the loaded data.">
+ <java classname="com.bigdata.journal.DumpJournal"
+ fork="true" failonerror="true">
+ <arg line="-pages ${lubm.journalFile}"/>
+ <classpath>
+ <path refid="runtime.classpath" />
+ </classpath>
+ </java>
+ </target>
+
+</project>
Added: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/results.txt
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/results.txt (rev 0)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-perf/lubm/results.txt 2014-07-13 14:54:11 UTC (rev 8543)
@@ -0,0 +1,96 @@
+
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6194" milliseconds="1295"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6530" milliseconds="1454"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5957" milliseconds="1151"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6974" milliseconds="1232"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6513" milliseconds="1028"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7520" milliseconds="1414"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6509" milliseconds="1386"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7574" milliseconds="1576"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7667" milliseconds="1732"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6904" milliseconds="1479"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6596" milliseconds="1245"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6866" milliseconds="1192"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7192" milliseconds="1469"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6873" milliseconds="1171"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7460" milliseconds="1646"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6328" milliseconds="1359"/>
+
+
+
+
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6989" milliseconds="2224"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6355" milliseconds="1948"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6148" milliseconds="1949"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6662" milliseconds="1958"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7632" milliseconds="2366"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7326" milliseconds="2255"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7159" milliseconds="2263"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5719" milliseconds="1837"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5979" milliseconds="1870"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7293" milliseconds="2176"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7381" milliseconds="2432"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7773" milliseconds="2326"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6357" milliseconds="1990"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6563" milliseconds="2203"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6886" milliseconds="2093"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6356" milliseconds="1966"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6958" milliseconds="2131"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6288" milliseconds="1894"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7433" milliseconds="2330"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6672" milliseconds="2118"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6582" milliseconds="1904"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7570" milliseconds="2342"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7620" milliseconds="2517"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6821" milliseconds="2167"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7752" milliseconds="2349"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5814" milliseconds="1926"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6910" milliseconds="2078"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7118" milliseconds="2184"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6793" milliseconds="2201"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7786" milliseconds="2404"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5993" milliseconds="1966"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6530" milliseconds="2125"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="5957" milliseconds="1928"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6974" milliseconds="2320"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6513" milliseconds="2225"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7520" milliseconds="2389"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6509" milliseconds="2080"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7574" milliseconds="2445"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7667" milliseconds="2490"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6904" milliseconds="2275"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6596" milliseconds="2250"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6866" milliseconds="2193"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7192" milliseconds="2395"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6873" milliseconds="2133"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="7460" milliseconds="2456"/>
+<?xml version="1.0"?><data modified="5639" milliseconds="1101"/><data modified="6328" milliseconds="2042"/>
+
+
+Run 64 - 8000
+3000 - ~1000ms
+5000 - ~1100ms
+5674 - 1300ms
+
+<?xml version="1.0"?><data modified="7783" milliseconds="1306"/>
+<?xml version="1.0"?><data modified="7578" milliseconds="1012"/>
+<?xml version="1.0"?><data modified="6326" milliseconds="735"/>
+<?xml version="1.0"?><data modified="7211" milliseconds="1029"/>
+<?xml version="1.0"?><data modified="7610" milliseconds="1072"/>
+<?xml version="1.0"?><data modified="7616" milliseconds="1134"/>
+<?xml version="1.0"?><data modified="6591" milliseconds="1087"/>
+<?xml version="1.0"?><data modified="6023" milliseconds="1107"/>
+<?xml version="1.0"?><data modified="7651" milliseconds="1251"/>
+<?xml version="1.0"?><data modified="6577" milliseconds="982"/>
+<?xml version="1.0"?><data modified="6091" milliseconds="877"/>
+<?xml version="1.0"?><data modified="6795" milliseconds="1030"/>
+<?xml version="1.0"?><data modified="6941" milliseconds="1257"/>
+<?xml version="1.0"?><data modified="7840" milliseconds="1275"/>
+<?xml version="1.0"?><data modified="6741" milliseconds="1212"/>
+<?xml version="1.0"?><data modified="7222" milliseconds="1234"/>
+<?xml version="1.0"?><data modified="5563" milliseconds="815"/>
+<?xml version="1.0"?><data modified="7391" milliseconds="1077"/>
+<?xml version="1.0"?><data modified="6123" milliseconds="911"/>
+<?xml version="1.0"?><data modified="7141" milliseconds="1249"/>
+<?xml version="1.0"?><data modified="6403" milliseconds="945"/>
+<?xml version="1.0"?><data modified="7952" milliseconds="1308"/>
Modified: branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsNoHistory.java
===================================================================
--- branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsNoHistory.java 2014-07-13 14:51:45 UTC (rev 8542)
+++ branches/BIGDATA_MGC_SMALL_SLOTS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsNoHistory.java 2014-07-13 14:54:11 UTC (rev 8543)
@@ -153,7 +153,7 @@
}
- public void test_multiple_csem_transaction_no_history_stress_readWriteTx()
+ public void Yes()
throws Exception {
final Random r = new Random();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-07-13 14:51:53
|
Revision: 8542
http://sourceforge.net/p/bigdata/code/8542
Author: martyncutcher
Date: 2014-07-13 14:51:45 +0000 (Sun, 13 Jul 2014)
Log Message:
-----------
branch to test/develop small slot allocator strategy to maximise allocation locality by only allocating form regions with "sparse" allocations therefore increasing the likelihood that small writes can be ellided to a single larger one, ideally covering a complete page, but in any case, reducing write IOPS.
Added Paths:
-----------
branches/BIGDATA_MGC_SMALL_SLOTS/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-13 12:25:35
|
Revision: 8541
http://sourceforge.net/p/bigdata/code/8541
Author: thompsonbry
Date: 2014-07-13 12:25:31 +0000 (Sun, 13 Jul 2014)
Log Message:
-----------
Javadoc for "REST API INSERT" does not handle .gz ticket.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-07-13 12:23:29 UTC (rev 8540)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-07-13 12:25:31 UTC (rev 8541)
@@ -270,8 +270,14 @@
conn = getUnisolatedConnection();
- /*
+ /**
* There is a request body, so let's try and parse it.
+ *
+ * FIXME This does not handle .gz or .zip files. We handle this
+ * in the
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/991" >REST API:
+ * INSERT does not handle .gz</a>
*/
final RDFParser rdfParser = rdfParserFactory.getParser();
@@ -466,10 +472,16 @@
/**
* There is a request body, so let's try and parse it.
*
- * <a href=
- * "https://sourceforge.net/apps/trac/bigdata/ticket/620"
- * > UpdateServlet fails to parse MIMEType when doing
- * conneg. </a>
+ * @see <a href=
+ * "https://sourceforge.net/apps/trac/bigdata/ticket/620"
+ * > UpdateServlet fails to parse MIMEType when
+ * doing conneg. </a>
+ *
+ * FIXME This does not handle .gz or .zip files. We
+ * handle this in the
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/991"
+ * >REST API: INSERT does not handle .gz</a>
*/
final String contentType = hconn.getContentType();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-13 12:23:33
|
Revision: 8540
http://sourceforge.net/p/bigdata/code/8540
Author: thompsonbry
Date: 2014-07-13 12:23:29 +0000 (Sun, 13 Jul 2014)
Log Message:
-----------
Looking back at the thread dump attached to the ticket, I see very little in the way of stack traces through bigdata. The only one in the property path code is this.
{{{
"com.bigdata.journal.Journal.executorService5" - Thread t@43
java.lang.Thread.State: RUNNABLE
at com.bigdata.bop.bindingSet.ListBindingSet.copy(ListBindingSet.java:290)
at com.bigdata.bop.bindingSet.ListBindingSet.<init>(ListBindingSet.java:267)
at com.bigdata.bop.bindingSet.ListBindingSet.clone(ListBindingSet.java:325)
at com.bigdata.bop.bindingSet.ListBindingSet.clone(ListBindingSet.java:43)
at com.bigdata.bop.paths.ArbitraryLengthPathOp$ArbitraryLengthPathTask.processChunk(ArbitraryLengthPathOp.java:511)
at com.bigdata.bop.paths.ArbitraryLengthPathOp$ArbitraryLengthPathTask.call(ArbitraryLengthPathOp.java:270)
at com.bigdata.bop.paths.ArbitraryLengthPathOp$ArbitraryLengthPathTask.call(ArbitraryLengthPathOp.java:196)
at java.util.concurrent.FutureTask.run(FutureTask.java:273)
at com.bigdata.bop.engine.ChunkedRunningQuery$ChunkTask.call(ChunkedRunningQuery.java:1281)
at com.bigdata.bop.engine.ChunkedRunningQuery$ChunkTaskWrapper.run(ChunkedRunningQuery.java:836)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:482)
at java.util.concurrent.FutureTask.run(FutureTask.java:273)
at com.bigdata.concurrent.FutureTaskMon.run(FutureTaskMon.java:63)
at com.bigdata.bop.engine.ChunkedRunningQuery$ChunkFutureTask.run(ChunkedRunningQuery.java:731)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1156)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:626)
at java.lang.Thread.run(Thread.java:804)
}}}
I've added tests for interrupts to two locations in the processChunk() code. One corresponds to the point where this stack trace passes through processChunk(). The other corresponds to the point where the initial solutions are flowing into the property path operator. Both check for an interrupt every 10 solutions.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/paths/ArbitraryLengthPathOp.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/paths/ArbitraryLengthPathOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/paths/ArbitraryLengthPathOp.java 2014-07-12 01:23:36 UTC (rev 8539)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/paths/ArbitraryLengthPathOp.java 2014-07-13 12:23:29 UTC (rev 8540)
@@ -52,6 +52,7 @@
import com.bigdata.bop.engine.AbstractRunningQuery;
import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.bop.join.JVMDistinctFilter;
import cutthecrap.utils.striterators.ICloseableIterator;
@@ -83,6 +84,10 @@
* solutions from the subquery with those in the parent context.
*
* @author <a href="mailto:mpe...@us...">Mike Personick</a>
+ *
+ * TODO There should be two version of this operator. One for the JVM
+ * heap and another for the native heap. This will help when large
+ * amounts of data are materialized by the internal collections.
*/
public class ArbitraryLengthPathOp extends PipelineOp {
@@ -187,6 +192,7 @@
}
+ @Override
public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
return new FutureTask<Void>(new ArbitraryLengthPathTask(this, context));
@@ -251,6 +257,7 @@
}
+ @Override
public Void call() throws Exception {
try {
@@ -346,10 +353,21 @@
}
if (!noInput) {
+
+ long chunksIn = 0L;
+
+ for (IBindingSet parentSolutionIn : chunkIn) {
+
+ /**
+ * @see <a href="http://trac.bigdata.com/ticket/865"
+ * >OutOfMemoryError instead of Timeout for SPARQL
+ * Property Paths </a>
+ */
+ if (chunksIn++ % 10 == 0 && Thread.interrupted()) {
+ throw new InterruptedException();
+ }
- for (IBindingSet parentSolutionIn : chunkIn) {
-
- final IConstant<?> key = joinVar != null ? parentSolutionIn.get(joinVar) : null;
+ final IConstant<?> key = joinVar != null ? parentSolutionIn.get(joinVar) : null;
if (log.isDebugEnabled()) {
log.debug("adding parent solution for joining: " + parentSolutionIn);
@@ -451,13 +469,16 @@
try {
- /*
- * TODO replace with code that does the PipelineJoins manually
- */
+ /*
+ * TODO Replace with code that does the PipelineJoins
+ * manually. Unrolling these iterations can be a major
+ * performance benefit. Another possibility is to use
+ * the GASEngine to expand the paths.
+ */
runningSubquery = queryEngine.eval(subquery,
nextRoundInput.toArray(new IBindingSet[nextRoundInput.size()]));
- long count = 0L;
+ long subqueryChunksOut = 0L; // #of chunks read from subquery
try {
// Declare the child query to the parent.
@@ -476,7 +497,14 @@
for (IBindingSet bs : chunk) {
- count++;
+ /**
+ * @see <a href="http://trac.bigdata.com/ticket/865"
+ * >OutOfMemoryError instead of Timeout for SPARQL
+ * Property Paths </a>
+ */
+ if (subqueryChunksOut++ % 10 == 0 && Thread.interrupted()) {
+ throw new InterruptedException();
+ }
if (log.isDebugEnabled()) {
log.debug("round " + i + " solution: " + bs);
@@ -532,7 +560,7 @@
if (log.isDebugEnabled()) {
log.debug("done with round " + i +
- ", count=" + count +
+ ", count=" + subqueryChunksOut +
", totalBefore=" + sizeBefore +
", totalAfter=" + solutionsOut.size() +
", totalNew=" + (solutionsOut.size() - sizeBefore));
@@ -1116,6 +1144,7 @@
}
+ @Override
public String toString() {
final StringBuilder sb = new StringBuilder();
@@ -1144,7 +1173,11 @@
}
/**
- * Lifted directly from the JVMDistinctBindingSetsOp.
+ * Lifted directly from the {@link JVMDistinctFilter}.
+ *
+ * TODO Refactor to use {@link JVMDistinctFilter} directly iff possible
+ * (e.g., a chain of the AALP operator followed by the DISTINCT
+ * solutions operator)
*/
private final static class SolutionKey {
@@ -1157,10 +1190,12 @@
this.hash = java.util.Arrays.hashCode(vals);
}
+ @Override
public int hashCode() {
return hash;
}
+ @Override
public boolean equals(final Object o) {
if (this == o)
return true;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-12 01:23:46
|
Revision: 8539
http://sourceforge.net/p/bigdata/code/8539
Author: tobycraig
Date: 2014-07-12 01:23:36 +0000 (Sat, 12 Jul 2014)
Log Message:
-----------
#843 - Fixed links on performance tab
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-12 01:12:41 UTC (rev 8538)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-12 01:23:36 UTC (rev 8539)
@@ -1485,11 +1485,20 @@
/* Performance */
-$('#tab-selector a[data-target=performance]').click(function(e) {
- $.get(RO_URL_PREFIX + 'counters', function(data) {
+$('#tab-selector a[data-target=performance]').click(loadPerformance);
+
+function loadPerformance(path) {
+ if(typeof(path) == 'undefined') {
+ path = '';
+ }
+ $.get(RO_URL_PREFIX + 'counters?' + path, function(data) {
$('#performance-tab .box').html(data);
+ $('#performance-tab .box a').click(function(e) {
+ e.preventDefault();
+ loadPerformance(this.href.split('?')[1]);
+ });
});
-});
+}
/* Utility functions */
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-12 01:12:49
|
Revision: 8538
http://sourceforge.net/p/bigdata/code/8538
Author: tobycraig
Date: 2014-07-12 01:12:41 +0000 (Sat, 12 Jul 2014)
Log Message:
-----------
Fixed queries/details links on status tab
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-11 23:54:19 UTC (rev 8537)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-12 01:12:41 UTC (rev 8538)
@@ -1346,8 +1346,8 @@
function getStatusNumbers(data) {
$('#status-text').html(data);
- $('#status-text a').eq(1).click(function(e) { e.preventDefault(); showQueries(false); return false; });
- $('#status-text a').eq(2).click(function(e) { e.preventDefault(); showQueries(true); return false; });
+ $('#status-text a').eq(-2).click(function(e) { e.preventDefault(); showQueries(false); return false; });
+ $('#status-text a').eq(-1).click(function(e) { e.preventDefault(); showQueries(true); return false; });
}
$('#show-queries').click(function(e) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-11 23:54:26
|
Revision: 8537
http://sourceforge.net/p/bigdata/code/8537
Author: tobycraig
Date: 2014-07-11 23:54:19 +0000 (Fri, 11 Jul 2014)
Log Message:
-----------
Fixed mime type typo
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-11 23:29:20 UTC (rev 8536)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-11 23:54:19 UTC (rev 8537)
@@ -779,7 +779,7 @@
var export_extensions = {
"application/rdf+xml": ['RDF/XML', 'rdf', true],
- "application/x-turtle": ['N-Triples', 'nt', true],
+ "application/n-triples": ['N-Triples', 'nt', true],
"application/x-turtle": ['Turtle', 'ttl', true],
"text/rdf+n3": ['N3', 'n3', true],
"application/trix": ['TriX', 'trix', true],
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-07-11 23:29:28
|
Revision: 8536
http://sourceforge.net/p/bigdata/code/8536
Author: tobycraig
Date: 2014-07-11 23:29:20 +0000 (Fri, 11 Jul 2014)
Log Message:
-----------
#961 - Added clone namespace and validation of namespace creation form
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-07-10 19:33:29 UTC (rev 8535)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-07-11 23:29:20 UTC (rev 8536)
@@ -244,11 +244,13 @@
<h1>Create namespace</h1>
<form id="namespace-create">
<label for="new-namespace-name">Name:</label> <input type="text" id="new-namespace-name"><br>
- <label for="new-namespace-index">Index:</label> <input type="checkbox" id="new-namespace-index"><br>
+ <label for="new-namespace-index">Full text index:</label> <input type="checkbox" id="new-namespace-index"><br>
<label for="new-namespace-truth-maintenance">Truth maintenance:</label> <input type="checkbox" id="new-namespace-truth-maintenance"><br>
<label for="new-namespace-quads">Quads:</label> <input type="checkbox" id="new-namespace-quads"><br>
+ <label for="new-namespace-rdr">RDR:</label> <input type="checkbox" id="new-namespace-rdr"><br>
<input type="submit" value="Create namespace">
</form>
+ <ul id="namespace-create-errors"></ul>
</div>
</div>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-10 19:33:29 UTC (rev 8535)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-07-11 23:29:20 UTC (rev 8536)
@@ -6,12 +6,12 @@
var PAGE_SIZE = 50, TOTAL_PAGES, CURRENT_PAGE;
var NAMESPACE_PARAMS = {
'name': 'com.bigdata.rdf.sail.namespace',
- 'index': 'com.bigdata.search.FullTextIndex.fieldsEnabled',
+ 'index': 'com.bigdata.rdf.store.AbstractTripleStore.textIndex',
'truthMaintenance': 'com.bigdata.rdf.sail.truthMaintenance',
- 'quads': 'com.bigdata.rdf.store.AbstractTripleStore.quads'
+ 'quads': 'com.bigdata.rdf.store.AbstractTripleStore.quads',
+ 'rdr': 'com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers'
};
-
CODEMIRROR_DEFAULTS = {
lineNumbers: true,
mode: 'sparql',
@@ -131,7 +131,7 @@
} else {
use = '<a href="#" class="use-namespace">Use</a>';
}
- $('#namespaces-list').append('<li data-name="' + title + '">' + titleText + ' - ' + use + ' - <a href="#" class="delete-namespace">Delete</a> - <a href="#" class="namespace-properties">Properties</a> - <a href="' + RO_URL_PREFIX + 'namespace/' + title + '/sparql" class="namespace-service-description">Service Description</a></li>');
+ $('#namespaces-list').append('<li data-name="' + title + '">' + titleText + ' - ' + use + ' - <a href="#" class="delete-namespace">Delete</a> - <a href="#" class="namespace-properties">Properties</a> - <a href="#" class="clone-namespace">Clone</a> - <a href="' + RO_URL_PREFIX + 'namespace/' + title + '/sparql" class="namespace-service-description">Service Description</a></li>');
}
$('.use-namespace').click(function(e) {
e.preventDefault();
@@ -152,6 +152,7 @@
$('.clone-namespace').click(function(e) {
e.preventDefault();
cloneNamespace($(this).parent().data('name'));
+ $('#namespace-create-errors').html('');
});
$('.namespace-service-description').click(function(e) {
return confirm('This can be an expensive operation. Proceed anyway?');
@@ -238,17 +239,45 @@
});
}
+function validateNamespaceOptions() {
+ var errors = [];
+ if(!$('#new-namespace-name').val().trim()) {
+ errors.push('Enter a name');
+ }
+ if($('#new-namespace-truth-maintenance').is(':checked') && $('#new-namespace-quads').is(':checked')) {
+ errors.push('You may not select both truth maintenance and quads');
+ }
+ if($('#new-namespace-rdr').is(':checked') && $('#new-namespace-quads').is(':checked')) {
+ errors.push('You may not select both RDR and quads');
+ }
+ $('#namespace-create-errors').html('');
+ for(var i=0; i<errors.length; i++) {
+ $('#namespace-create-errors').append('<li>' + errors[i] + '</li>');
+ }
+ var valid = errors.length == 0;
+ if(valid) {
+ $('#namespace-create input[type=submit]').removeAttr('disabled');
+ } else {
+ $('#namespace-create input[type=submit]').attr('disabled', 'disabled');
+ }
+ return errors.length == 0;
+}
+
+$('#namespace-create input').change(validateNamespaceOptions);
+$('#namespace-create input').keyup(validateNamespaceOptions);
+
function createNamespace(e) {
e.preventDefault();
+ if(!validateNamespaceOptions()) {
+ return;
+ }
// get new namespace name and config options
var params = {};
params.name = $('#new-namespace-name').val().trim();
- if(!params.name) {
- return;
- }
params.index = $('#new-namespace-index').is(':checked');
params.truthMaintenance = $('#new-namespace-truth-maintenance').is(':checked');
params.quads = $('#new-namespace-quads').is(':checked');
+ params.rdr = $('#new-namespace-rdr').is(':checked');
// TODO: validate namespace
// TODO: allow for other options to be specified
var data = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">\n<properties>\n';
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-07-10 19:33:32
|
Revision: 8535
http://sourceforge.net/p/bigdata/code/8535
Author: thompsonbry
Date: 2014-07-10 19:33:29 +0000 (Thu, 10 Jul 2014)
Log Message:
-----------
Added a generalized case for arbitrary application logic for stored queries.
There is also a special case for a parameterized SPARQL query.
See #989 (Stored query service)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/stored-query-002.rq
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/service/storedquery/TestStoredQueryService.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java 2014-07-10 18:35:39 UTC (rev 8534)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/storedquery/StoredQueryService.java 2014-07-10 19:33:29 UTC (rev 8535)
@@ -54,15 +54,16 @@
* a SPARQL query or arbitrary procedural application logic, but it must
* evaluate to a solution multi-set. The service interface is written to the
* openrdf interfaces in order to remove the burden of dealing with bigdata
- * {@link IV}s from the application. The effective value of the baseURI during
- * query evaluation will be the SERVICE URI.
+ * {@link IV}s from the application.
* <p>
* In order to use a stored query, a concrete instance of this class must be
- * registered against the {@link ServiceRegistry}:
+ * registered against the {@link ServiceRegistry}. The choice of the SERVICE URI
+ * is up to the application. The effective value of the baseURI during query
+ * evaluation will be the SERVICE URI.
*
* <pre>
- * final URI serviceURI = new URIImpl(StoredQueryService.Options.NAMESPACE
- * + "my-service");
+ * final URI serviceURI = new URIImpl(
+ * "http://www.bigdata.com/rdf/stored-query#my-stored-query");
*
* ServiceRegistry.getInstance().add(serviceURI, new MyStoredQueryService());
* </pre>
@@ -72,7 +73,7 @@
*
* <pre>
* SELECT * {
- * SERVICE <http://www.bigdata.com/rdf/stored-query#my-service> { }
+ * SERVICE <http://www.bigdata.com/rdf/stored-query#my-stored-query> { }
* }
* </pre>
*
@@ -82,7 +83,7 @@
*
* <pre>
* SELECT * {
- * SERVICE <http://www.bigdata.com/rdf/stored-query#my-service> {
+ * SERVICE <http://www.bigdata.com/rdf/stored-query#my-stored-query> {
* bd:serviceParam :color :"blue" .
* bd:serviceParam :color :"green" .
* bd:serviceParam :size :"large" .
@@ -101,48 +102,24 @@
*
* @see <a href="http://trac.bigdata.com/ticket/989">Stored Query Service</a>
*
- * TODO Wiki page.
+ * FIXME Wiki page.
*
- * TODO Implicit prefix declaration for bsq.
+ * FIXME Generalize to support groovy scripting.
*
- * TODO Why does this work?
- *
- * <pre>
- * SELECT ?book ?title ?price
- * {
- * SERVICE <http://www.bigdata.com/rdf/stored-query#test_stored_query_001> {
- * }
- * }
- * </pre>
- *
- * while this does not work
- *
- * <pre>
- * PREFIX bsq: <http://www.bigdata.com/rdf/stored-query#>
- *
- * SELECT ?book ?title ?price
- * {
- * SERVICE <bsq#test_stored_query_001> {
- * }
- * }
- * </pre>
- *
* TODO We could use {@link ASTEvalHelper} to evaluate at the bigdata level
* without forcing the materialization of any variable bindings from the
* lexicon indices. This would be faster for some purposes, especially if
* the stored procedure is only used to JOIN into an outer query as in
* <code>SELECT * { SERVICE bsq:my-service {} }</code>
- *
- * FIXME Generalize to support groovy scripting.
*/
abstract public class StoredQueryService implements ServiceFactory {
public interface Options {
- /**
- * The namespace used for stored query service.
- */
- String NAMESPACE = "http://www.bigdata.com/rdf/stored-query#";
+// /**
+// * The namespace used for stored query service.
+// */
+// String NAMESPACE = "http://www.bigdata.com/rdf/stored-query#";
}
@@ -207,7 +184,7 @@
*
* @param cxn
* The connection that should be used to read on the SPARQL
- * database.
+ * database. The connection will be closed by the caller.
* @param createParams
* The SERVICE creation parameters.
* @param serviceParams
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2014-07-10 18:35:39 UTC (rev 8534)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2014-07-10 19:33:29 UTC (rev 8535)
@@ -42,7 +42,6 @@
* A vocabulary for bigdata specific extensions.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface BD {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/stored-query-002.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/stored-query-002.rq 2014-07-10 18:35:39 UTC (rev 8534)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/stored-query-002.rq 2014-07-10 19:33:29 UTC (rev 8535)
@@ -1,4 +1,3 @@
-PREFIX bsq: <http://www.bigdata.com/rdf/stored-query#>
PREFIX : <http://example.org/book/>
SELECT ?book ?title ?price
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/service/storedquery/TestStoredQueryService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/service/storedquery/TestStoredQueryService.java 2014-07-10 18:35:39 UTC (rev 8534)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/service/storedquery/TestStoredQueryService.java 2014-07-10 19:33:29 UTC (rev 8535)
@@ -60,6 +60,8 @@
super(name);
}
+ static private final String NAMESPACE = "http://www.bigdata.com/rdf/stored-query#";
+
/**
* Simple stored query test. Note that this test also verifies that the
* BINDINGS flow into the stored query.
@@ -100,7 +102,7 @@
}
- final URI serviceURI = new URIImpl(StoredQueryService.Options.NAMESPACE + getName());
+ final URI serviceURI = new URIImpl( NAMESPACE + getName());
try {
// register the service.
@@ -167,7 +169,7 @@
}
- final URI serviceURI = new URIImpl(StoredQueryService.Options.NAMESPACE + getName());
+ final URI serviceURI = new URIImpl(NAMESPACE+ getName());
try {
// register the service.
@@ -244,7 +246,7 @@
}
- final URI serviceURI = new URIImpl(StoredQueryService.Options.NAMESPACE + getName());
+ final URI serviceURI = new URIImpl(NAMESPACE+ getName());
try {
// register the service.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
