This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <mrp...@us...> - 2014-09-02 15:01:47
|
Revision: 8635
http://sourceforge.net/p/bigdata/code/8635
Author: mrpersonick
Date: 2014-09-02 15:01:44 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Ticket #714: Sesame 2.7. Committing Bryan's changes for inline VALUES.
Modified Paths:
--------------
branches/SESAME_2_7/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataComplexSparqlQueryTest.java
Modified: branches/SESAME_2_7/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java
===================================================================
--- branches/SESAME_2_7/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -49,6 +49,7 @@
import com.bigdata.relation.accesspath.UnsyncLocalOutputBuffer;
import cutthecrap.utils.striterators.ICloseableIterator;
+import cutthecrap.utils.striterators.SingleValueIterator;
/**
* Operator builds a hash index from the source solutions. Once all source
@@ -100,6 +101,17 @@
*/
final String NAMED_SET_SOURCE_REF = "namedSetSourceRef";
+ /**
+ * An optional attribute specifying the <em>source</em> IBindingSet[]
+ * for the index build operation. Normally, the hash index is built from
+ * the solutions flowing through the pipeline. When this attribute is
+ * specified, the hash index is instead built from the solutions in the
+ * specified IBindingSet[]. Regardless, the solutions flowing through
+ * the pipeline are copied to the sink once the hash index has been
+ * built.
+ */
+ final String BINDING_SETS_SOURCE = "bindingSets";
+
}
/**
@@ -138,20 +150,10 @@
BOp.Annotations.EVALUATION_CONTEXT + "="
+ getEvaluationContext());
}
-// if (getEvaluationContext() != BOpEvaluationContext.CONTROLLER) {
-// throw new IllegalArgumentException(
-// BOp.Annotations.EVALUATION_CONTEXT + "="
-// + getEvaluationContext());
-// }
-
- if (getMaxParallel() != 1) {
- /*
- * Parallel evaluation is not allowed. This operator writes on an
- * object that is not thread-safe for mutation.
- */
+ if (getEvaluationContext() != BOpEvaluationContext.CONTROLLER) {
throw new IllegalArgumentException(
- PipelineOp.Annotations.MAX_PARALLEL + "="
- + getMaxParallel());
+ BOp.Annotations.EVALUATION_CONTEXT + "="
+ + getEvaluationContext());
}
if (!isLastPassRequested()) {
@@ -224,11 +226,11 @@
}
- /**
- * Evaluates the subquery for each source binding set. If the controller
- * operator is interrupted, then the subqueries are cancelled. If a subquery
- * fails, then all subqueries are cancelled.
- */
+ /**
+ * Evaluates the subquery for each source binding set. If the controller
+ * operator is interrupted, then the subqueries are cancelled. If a subquery
+ * fails, then all subqueries are cancelled.
+ */
private static class ChunkTask implements Callable<Void> {
private final BOpContext<IBindingSet> context;
@@ -268,7 +270,7 @@
this.stats = ((NamedSolutionSetStats) context.getStats());
- // Metadata to identify the named solution set.
+ // Metadata to identify the target named solution set.
final INamedSolutionSetRef namedSetRef = (INamedSolutionSetRef) op
.getRequiredProperty(Annotations.NAMED_SET_REF);
@@ -312,8 +314,10 @@
}
// true iff we will build the index from the pipeline.
- this.sourceIsPipeline = null == op
- .getProperty(Annotations.NAMED_SET_SOURCE_REF);
+ this.sourceIsPipeline //
+ = (op.getProperty(Annotations.NAMED_SET_SOURCE_REF) == null)
+ && (op.getProperty(Annotations.BINDING_SETS_SOURCE) == null)
+ ;
}
@@ -390,7 +394,7 @@
src = context.getSource();
- } else {
+ } else if (op.getProperty(Annotations.NAMED_SET_SOURCE_REF) != null) {
/*
* Metadata to identify the optional *source* solution set. When
@@ -403,6 +407,22 @@
src = context.getAlternateSource(namedSetSourceRef);
+ } else if (op.getProperty(Annotations.BINDING_SETS_SOURCE) != null) {
+
+ /*
+ * The IBindingSet[] is directly given. Just wrap it up as an
+ * iterator. It will visit a single chunk of solutions.
+ */
+ final IBindingSet[] bindingSets = (IBindingSet[]) op
+ .getProperty(Annotations.BINDING_SETS_SOURCE);
+
+ src = new SingleValueIterator<IBindingSet[]>(bindingSets);
+
+ } else {
+
+ throw new UnsupportedOperationException(
+ "Source was not specified");
+
}
try {
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -166,6 +166,7 @@
}
+ @Override
public String toString(final int indent) {
final LinkedHashSet<IVariable<?>> declaredVars = getDeclaredVariables();
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -148,6 +148,7 @@
}
+ @Override
public String toString() {
final StringBuilder sb = new StringBuilder();
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -2125,7 +2125,51 @@
}
+ /**
+ * Return the join variables for a VALUES clause (embedded only - not
+ * top-level).
+ *
+ * @param bc The VALUES clause (a bunch of solutions)
+ * @param stats A static analysis of those solutions.
+ * @param vars
+ * @return
+ */
+ public Set<IVariable<?>> getJoinVars(final BindingsClause bc,
+ final ISolutionSetStats stats,
+ final Set<IVariable<?>> vars) {
+ /*
+ * The variables which will be definitely bound based on the solutions
+ * in the VALUES clause.
+ *
+ * Note: Collection is not modifyable, so we copy it.
+ */
+ final Set<IVariable<?>> boundByBindingsClause = new LinkedHashSet<IVariable<?>>(
+ stats.getAlwaysBound());
+
+ /*
+ * The variables which are definitely bound on entry to the join group
+ * in which the VALUES clause appears.
+ */
+ final Set<IVariable<?>> incomingBindings = getDefinitelyIncomingBindings(
+ bc, new LinkedHashSet<IVariable<?>>());
+
+ /*
+ * This is only those variables which are bound on entry into the group
+ * in which the VALUES join appears *and* which are "must" bound
+ * variables projected by the VALUES.
+ *
+ * FIXME Is this the correct semantics? I followed the pattern for SERVICE.
+ */
+ boundByBindingsClause.retainAll(incomingBindings);
+
+ vars.addAll(boundByBindingsClause);
+
+ return vars;
+
+ }
+
+
/**
* Return the join variables for an INCLUDE of a pre-existing named solution
* set.
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -61,6 +61,7 @@
import com.bigdata.bop.join.JoinAnnotations;
import com.bigdata.bop.join.JoinTypeEnum;
import com.bigdata.bop.join.NestedLoopJoinOp;
+import com.bigdata.bop.join.SolutionSetHashJoinOp;
import com.bigdata.bop.paths.ArbitraryLengthPathOp;
import com.bigdata.bop.paths.ZeroLengthPathOp;
import com.bigdata.bop.rdf.join.ChunkedMaterializationOp;
@@ -90,7 +91,6 @@
import com.bigdata.rdf.internal.constraints.INeedsMaterialization.Requirement;
import com.bigdata.rdf.internal.constraints.InBOp;
import com.bigdata.rdf.internal.constraints.IsBoundBOp;
-import com.bigdata.rdf.internal.constraints.NowBOp;
import com.bigdata.rdf.internal.constraints.ProjectedConstraint;
import com.bigdata.rdf.internal.constraints.SPARQLConstraint;
import com.bigdata.rdf.internal.constraints.TryBeforeMaterializationConstraint;
@@ -154,6 +154,7 @@
import com.bigdata.rdf.spo.SPOPredicate;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.relation.accesspath.ElementFilter;
+import com.bigdata.striterator.Chunkerator;
import cutthecrap.utils.striterators.FilterBase;
import cutthecrap.utils.striterators.IFilter;
@@ -1469,14 +1470,184 @@
}
/**
- * TODO Grab the binding sets from the BindingsClause, attach them to
- * the query as a named subquery with a hash index, and then add a
- * named subquery include to the pipeline right here.
+ * This handles a VALUES clause. It grabs the binding sets from the
+ * BindingsClause, attach them to the query as a named subquery with a hash
+ * index, and then add a named subquery include to the pipeline right here.
+ * <p>
+ * The VALUES are interpreted using a solution set hash join. The "plan" for
+ * the hash join of the VALUES with the solutions flowing through the
+ * pipeline is: (a) we take the IBindingSet[] and use a {@link HashIndexOp}
+ * to generate the hash index; and (b) we use a
+ * {@link SolutionSetHashJoinOp} to join the solutions from the pipeline
+ * with those in the hash index. Both JVM and HTree versions of this plan
+ * are supported.
+ * <p>
+ * 1. {@link HashIndexOp} (JVM or HTree): Specify the IBindingSet[] as the
+ * source. When the HashIndexOp runs, it will build a hash index from the
+ * IBindingSet[].
+ * <p>
+ * Note: The join variables need to be set based on the known bound
+ * variables in the context where we will evaluate the solution set hash
+ * join (head of the sub-SELECT, OPTIONAL) and those that are bound by the
+ * solution set hash join.
+ * <p>
+ * Note: The static analysis code needs to examine the definitely, and maybe
+ * produced bindings for the {@link BindingsClause}. See the
+ * {@link ISolutionSetStats} interface and
+ * {@link SolutionSetStatserator#get(IBindingSet[])} for a convenience
+ * method.
+ * <p>
+ * 2. {@link SolutionSetHashJoinOp} (JVM or HTree): Joins the solutions
+ * flowing into the sub-query or update with the solutions from the
+ * HashIndexOp. This will take each solution from the pipeline, probe the
+ * hash index for solutions specified by the VALUES clause, and then do a
+ * JOIN for each such solution that is discovered.
*/
private static PipelineOp addValues(PipelineOp left,
- final BindingsClause bc,
+ final BindingsClause bindingsClause,
final Set<IVariable<?>> doneSet, final AST2BOpContext ctx) {
+ // Convert solutions from VALUES clause to an IBindingSet[].
+ final IBindingSet[] bindingSets = BOpUtility.toArray(
+ new Chunkerator<IBindingSet>(bindingsClause.getBindingSets().iterator()),//
+ null/*stats*/
+ );
+
+ // Static analysis of the VALUES solutions.
+ final ISolutionSetStats bindingsClauseStats = SolutionSetStatserator
+ .get(bindingSets);
+
+ @SuppressWarnings("rawtypes")
+ final Map<IConstraint, Set<IVariable<IV>>> needsMaterialization = new LinkedHashMap<IConstraint, Set<IVariable<IV>>>();
+
+ /*
+ * BindingsClause is an IBindingsProducer, but it should also be
+ * an IJoinNode. That will let us attach constraints
+ * (getJoinConstraints()) and identify the join variables for the VALUES
+ * sub-plan (getJoinVars()).
+ */
+ final IConstraint[] joinConstraints = getJoinConstraints(
+ getJoinConstraints(bindingsClause), needsMaterialization);
+
+ /*
+ * Model the VALUES JOIN by building a hash index over the IBindingSet[]
+ * from the VALUES clause. Then use a solution set hash join to join the
+ * solutions flowing through the pipeline with those in the hash index.
+ */
+ final String solutionSetName = "--values-" + ctx.nextId(); // Unique name.
+
+ final Set<IVariable<?>> joinVarSet = ctx.sa.getJoinVars(bindingsClause,
+ bindingsClauseStats, new LinkedHashSet<IVariable<?>>());
+
+ @SuppressWarnings("rawtypes")
+ final IVariable[] joinVars = joinVarSet.toArray(new IVariable[0]);
+
+// if (joinVars.length == 0) {
+//
+// /*
+// * Note: If there are no join variables then the join will
+// * examine the full N x M cross product of solutions. That is
+// * very inefficient, so we are logging a warning.
+// */
+//
+// log.warn("No join variables: " + subqueryRoot);
+//
+// }
+
+ final INamedSolutionSetRef namedSolutionSet = NamedSolutionSetRefUtility.newInstance(
+ ctx.queryId, solutionSetName, joinVars);
+
+ // VALUES is not optional.
+ final JoinTypeEnum joinType = JoinTypeEnum.Normal;
+
+ // lastPass is required except for normal joins.
+ final boolean lastPass = false;
+
+ // true if we will release the HTree as soon as the join is done.
+ // Note: also requires lastPass.
+ final boolean release = lastPass;
+
+ // join can be pipelined unless last pass evaluation is required
+ final int maxParallel = lastPass ? 1
+ : ctx.maxParallelForSolutionSetHashJoin;
+
+ // Generate the hash index operator.
+ if(ctx.nativeHashJoins) {
+ left = applyQueryHints(new HTreeHashIndexOp(leftOrEmpty(left),//
+ new NV(BOp.Annotations.BOP_ID, ctx.nextId()),//
+ new NV(BOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ new NV(PipelineOp.Annotations.MAX_PARALLEL, 1),// required for lastPass
+ new NV(PipelineOp.Annotations.LAST_PASS, true),// required
+ new NV(PipelineOp.Annotations.SHARED_STATE, true),// live stats.
+ new NV(HTreeHashIndexOp.Annotations.RELATION_NAME, new String[]{ctx.getLexiconNamespace()}),// new NV(HTreeHashIndexOp.Annotations.JOIN_VARS, joinVars),//
+ new NV(HTreeHashIndexOp.Annotations.JOIN_TYPE, joinType),//
+ new NV(HTreeHashIndexOp.Annotations.JOIN_VARS, joinVars),//
+ new NV(HTreeHashIndexOp.Annotations.CONSTRAINTS, joinConstraints),// Note: will be applied by the solution set hash join.
+// new NV(HTreeHashIndexOp.Annotations.SELECT, projectedVars),//
+ new NV(HTreeHashIndexOp.Annotations.BINDING_SETS_SOURCE, bindingSets),// source solutions from VALUES.
+ new NV(HTreeHashIndexOp.Annotations.NAMED_SET_REF, namedSolutionSet)// output named solution set.
+ ), bindingsClause, ctx);
+ } else {
+ left = applyQueryHints(new JVMHashIndexOp(leftOrEmpty(left),//
+ new NV(BOp.Annotations.BOP_ID, ctx.nextId()),//
+ new NV(BOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ new NV(PipelineOp.Annotations.MAX_PARALLEL, 1),// required for lastPass
+ new NV(PipelineOp.Annotations.LAST_PASS, true),// required
+ new NV(PipelineOp.Annotations.SHARED_STATE, true),// live stats.
+ new NV(JVMHashIndexOp.Annotations.JOIN_TYPE, joinType),//
+ new NV(JVMHashIndexOp.Annotations.JOIN_VARS, joinVars),//
+ new NV(JVMHashIndexOp.Annotations.CONSTRAINTS, joinConstraints),// Note: will be applied by the solution set hash join.
+// new NV(HTreeHashIndexOp.Annotations.SELECT, projectedVars),//
+ new NV(HTreeHashIndexOp.Annotations.BINDING_SETS_SOURCE, bindingSets),// source solutions from VALUES.
+ new NV(JVMHashIndexOp.Annotations.NAMED_SET_REF, namedSolutionSet)// output named solution set.
+ ), bindingsClause, ctx);
+ }
+
+ // Generate the solution set hash join operator.
+ if(ctx.nativeHashJoins) {
+ left = applyQueryHints(new HTreeSolutionSetHashJoinOp(
+ leftOrEmpty(left),//
+ new NV(BOp.Annotations.BOP_ID, ctx.nextId()),//
+ new NV(BOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ new NV(PipelineOp.Annotations.MAX_PARALLEL, maxParallel),//
+ new NV(PipelineOp.Annotations.SHARED_STATE, true),// live stats.
+// new NV(HTreeSolutionSetHashJoinOp.Annotations.OPTIONAL, optional),//
+// new NV(HTreeSolutionSetHashJoinOp.Annotations.JOIN_VARS, joinVars),//
+// new NV(HTreeSolutionSetHashJoinOp.Annotations.SELECT, null/*all*/),//
+// new NV(HTreeSolutionSetHashJoinOp.Annotations.CONSTRAINTS, joinConstraints),//
+ new NV(HTreeSolutionSetHashJoinOp.Annotations.RELEASE, release),//
+ new NV(HTreeSolutionSetHashJoinOp.Annotations.LAST_PASS, lastPass),//
+ new NV(HTreeSolutionSetHashJoinOp.Annotations.NAMED_SET_REF, namedSolutionSet)//
+ ), bindingsClause, ctx);
+ } else {
+ left = applyQueryHints(new JVMSolutionSetHashJoinOp(
+ leftOrEmpty(left),//
+ new NV(BOp.Annotations.BOP_ID, ctx.nextId()),//
+ new NV(BOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ new NV(PipelineOp.Annotations.MAX_PARALLEL, maxParallel),//
+ new NV(PipelineOp.Annotations.SHARED_STATE, true),// live stats.
+// new NV(JVMSolutionSetHashJoinOp.Annotations.OPTIONAL, optional),//
+// new NV(JVMSolutionSetHashJoinOp.Annotations.JOIN_VARS, joinVars),//
+// new NV(JVMSolutionSetHashJoinOp.Annotations.SELECT, null/*all*/),//
+// new NV(JVMSolutionSetHashJoinOp.Annotations.CONSTRAINTS, joinConstraints),//
+ new NV(JVMSolutionSetHashJoinOp.Annotations.RELEASE, release),//
+ new NV(JVMSolutionSetHashJoinOp.Annotations.LAST_PASS, lastPass),//
+ new NV(JVMSolutionSetHashJoinOp.Annotations.NAMED_SET_REF, namedSolutionSet)//
+ ), bindingsClause, ctx);
+ }
+
+ /*
+ * For each filter which requires materialization steps, add the
+ * materializations steps to the pipeline and then add the filter to the
+ * pipeline.
+ */
+ left = addMaterializationSteps3(left, doneSet, needsMaterialization,
+ bindingsClause.getQueryHints(), ctx);
+
return left;
}
@@ -2672,7 +2843,7 @@
continue;
} else if (child instanceof BindingsClause) {
/*
- * VALUES clause
+ * FIXME Support VALUES clause
*/
left = addValues(left,
(BindingsClause) child, doneSet, ctx);
Modified: branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataComplexSparqlQueryTest.java
===================================================================
--- branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataComplexSparqlQueryTest.java 2014-09-02 14:36:42 UTC (rev 8634)
+++ branches/SESAME_2_7/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataComplexSparqlQueryTest.java 2014-09-02 15:01:44 UTC (rev 8635)
@@ -29,12 +29,17 @@
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.log4j.Logger;
import org.junit.Test;
+import org.openrdf.model.Literal;
import org.openrdf.model.Resource;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQueryResult;
import org.openrdf.query.parser.sparql.ComplexSPARQLQueryTest;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryException;
@@ -42,11 +47,14 @@
import org.openrdf.rio.RDFParseException;
import com.bigdata.BigdataStatics;
+import com.bigdata.bop.BOpUtility;
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.IIndexManager;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.sail.BigdataSailRepository;
+import com.bigdata.rdf.sail.BigdataSailTupleQuery;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpUtility;
/**
* Bigdata integration for the {@link ComplexSPARQLQueryTest}.
@@ -301,11 +309,55 @@
super.testSES1991RANDEvaluation();
}
+ /**
+ * The one is fixed now.
+ */
@Override
@Test
public void testValuesInOptional() throws Exception {
- if (runKnownBadTests)
super.testValuesInOptional();
}
+
+ /**
+ * TODO Write optimizer to pull this BindingsClause out of the join
+ * group and make it global.
+ */
+ public void testRequiredValues() throws Exception {
+ loadTestData("/testdata-query/dataset-ses1692.trig");
+ StringBuilder query = new StringBuilder();
+ query.append(" PREFIX : <http://example.org/>\n");
+ query.append(" SELECT DISTINCT ?a ?name ?isX WHERE { ?b :p1 ?a . ?a :name ?name. ?a a :X . VALUES(?isX) { (:X) } } ");
+ BigdataSailTupleQuery tq = (BigdataSailTupleQuery)
+ conn.prepareTupleQuery(QueryLanguage.SPARQL, query.toString());
+
+ if (logger.isInfoEnabled()) {
+ logger.info("optimized ast:\n"+tq.optimize());
+ logger.info("query plan:\n"+BOpUtility.toString(tq.getASTContainer().getQueryPlan()));
+ }
+
+ TupleQueryResult result = tq.evaluate();
+ assertNotNull(result);
+ assertTrue(result.hasNext());
+
+ int count = 0;
+ while (result.hasNext()) {
+ count++;
+ BindingSet bs = result.next();
+ System.out.println(bs);
+ URI a = (URI)bs.getValue("a");
+ assertNotNull(a);
+ Value isX = bs.getValue("isX");
+ Literal name = (Literal)bs.getValue("name");
+ assertNotNull(name);
+ if (a.stringValue().endsWith("a1")) {
+ assertNotNull(isX);
+ }
+ else if (a.stringValue().endsWith(("a2"))) {
+ assertNull(isX);
+ }
+ }
+ assertEquals(1, count);
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-09-02 14:36:45
|
Revision: 8634
http://sourceforge.net/p/bigdata/code/8634
Author: martyncutcher
Date: 2014-09-02 14:36:42 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Undo previous commit r8032, prior to re-syncing post-review
Revision Links:
--------------
http://sourceforge.net/p/bigdata/code/8032
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Dec 19, 2006
+ *
*/
package com.bigdata.btree;
@@ -36,14 +37,9 @@
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.FutureTask;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -92,7 +88,6 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Split;
import com.bigdata.util.InnerCause;
-import com.bigdata.util.StackInfoReport;
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
@@ -255,8 +250,6 @@
* mutation.
*/
final protected boolean readOnly;
-
- final ReentrantReadWriteLock lock;
/**
* Optional cache for {@link INodeData} and {@link ILeafData} instances and
@@ -967,8 +960,6 @@
this.store = store;
this.readOnly = readOnly;
-
- this.lock = UnisolatedReadWriteIndex.getReadWriteLock(this);
// /*
// * The Memoizer is not used by the mutable B+Tree since it is not safe
@@ -1986,8 +1977,6 @@
}
};
-
- volatile Throwable error;
final public Object insert(Object key, Object value) {
@@ -3380,14 +3369,7 @@
}
- // If a mutable tree and a read-only operation then ensure there are no evictions from this thread
- final Integer rcount = threadLockMap.get(Thread.currentThread().getId());
- if (!isReadOnly() && rcount != null /* && false this should stochastically fail!*/) {
- // NOP
- assert rcount.intValue() > 0;
- } else {
- doSyncTouch(node);
- }
+ doSyncTouch(node);
}
@@ -3451,7 +3433,7 @@
*/
// assert isReadOnly() || ndistinctOnWriteRetentionQueue > 0;
-
+
node.referenceCount++;
if (!writeRetentionQueue.add(node)) {
@@ -3616,15 +3598,6 @@
}
- private void badNode(final AbstractNode<?> node) {
-// try {
-// Thread.sleep(50);
-// } catch (InterruptedException e) {
-// // ignore;
-// }
- throw new AssertionError("ReadOnly and identity: " + node.identity);
- }
-
/**
* Codes the node and writes the coded record on the store (non-recursive).
* The node MUST be dirty. If the node has a parent, then the parent is
@@ -3644,31 +3617,7 @@
* @return The persistent identity assigned by the store.
*/
protected long writeNodeOrLeaf(final AbstractNode<?> node) {
-
- /*
- * The check for the writeLock held on an update is currently disabled since
- * a number of Unit tests failed with it active. The exception information
- * indicates that the writeLock is not held in any of the test failure scenarios, so
- * the thread check does not fail due to submitted tasks.
- * TODO: we need to clarify the requirements for holding the writeLock across any update
- */
- if (false && !lock.isWriteLockedByCurrentThread()) {
- throw new AssertionError("WriteLock not held, readOnly: " + this.isReadOnly() + ", locked by other: " + lock.isWriteLocked());
- }
-
- if (error != null) {
- throw new IllegalStateException("BTree is in an error state", error);
- }
-
-// if (node.writing != null) {
-// final Throwable remote = node.writing;
-// node.writing = null;
-//
-// throw new AssertionError("Duplicate Call - ", remote);
-// }
-//
- // node.writing = new StackInfoReport("Write Guard");
-
+
assert root != null; // i.e., isOpen().
assert node != null;
assert node.btree == this;
@@ -3692,10 +3641,7 @@
* TestMROWTransactions might also demonstrate an issue
* occasionally. If so, then check for the same root cause.
*/
- if (node.isReadOnly()) {
- badNode(node); // supports debugging
- }
- assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
+ assert !node.isReadOnly();
assertNotReadOnly();
/*
@@ -3795,14 +3741,6 @@
// No longer dirty (prevents re-coding on re-eviction).
node.setDirty(false);
-// if (node.writing == null) {
-// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
-//
-// throw new AssertionError("Concurrent modification of thread guard");
-// }
-
-// node.writing = null;
-
return 0L;
}
@@ -3892,14 +3830,6 @@
}
-// if (node.writing == null) {
-// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
-//
-// throw new AssertionError("Concurrent modification of thread guard");
-// }
-//
-// node.writing = null;
-
return addr;
}
@@ -3926,6 +3856,40 @@
if (addr == IRawStore.NULL)
throw new IllegalArgumentException();
+// final Long addr2 = Long.valueOf(addr);
+//
+// if (storeCache != null) {
+//
+// // test cache : will touch global LRU iff found.
+// final IAbstractNodeData data = (IAbstractNodeData) storeCache
+// .get(addr);
+//
+// if (data != null) {
+//
+// // Node and Leaf MUST NOT make it into the global LRU or store
+// // cache!
+// assert !(data instanceof AbstractNode<?>);
+//
+// final AbstractNode<?> node;
+//
+// if (data.isLeaf()) {
+//
+// node = nodeSer.nodeFactory.allocLeaf(this, addr,
+// (ILeafData) data);
+//
+// } else {
+//
+// node = nodeSer.nodeFactory.allocNode(this, addr,
+// (INodeData) data);
+//
+// }
+//
+// // cache hit.
+// return node;
+//
+// }
+//
+// }
final ByteBuffer tmp;
{
@@ -3982,6 +3946,21 @@
}
+// if (storeCache != null) {
+//
+// // update cache : will touch global LRU iff cache is modified.
+// final IAbstractNodeData data2 = (IAbstractNodeData) storeCache
+// .putIfAbsent(addr2, data);
+//
+// if (data2 != null) {
+//
+// // concurrent insert, use winner's value.
+// data = data2;
+//
+// }
+//
+// }
+
// wrap as Node or Leaf.
final AbstractNode<?> node = nodeSer.wrap(this, addr, data);
@@ -4325,35 +4304,4 @@
}
-
- /**
- * Maintain count of readLocks on this Thread. Use ThreadLocal for now, but
- * could switch to HashMap if necessary
- */
-// ThreadLocal<AtomicInteger> threadReadLockCount = new ThreadLocal<AtomicInteger>();
- ConcurrentHashMap<Long, Integer> threadLockMap = new ConcurrentHashMap<Long, Integer>();
-
- public void readLockedThread() {
-// final AtomicInteger ai = threadReadLockCount.get();
-// if (ai == null) {
-// threadReadLockCount.set(new AtomicInteger(1));
-// } else {
-// ai.incrementAndGet();
-// }
- final long thisThreadId = Thread.currentThread().getId();
- final Integer entry = threadLockMap.get(thisThreadId);
- final Integer newVal = entry == null ? 1 : 1 + entry.intValue();
- threadLockMap.put(thisThreadId, newVal);
- }
-
- public void readUnlockedThread() {
- final long thisThreadId = Thread.currentThread().getId();
- final Integer entry = threadLockMap.get(thisThreadId);
- assert entry != null;
- if (entry.intValue() == 1) {
- threadLockMap.remove(thisThreadId);
- } else {
- threadLockMap.put(thisThreadId, entry.intValue() - 1);
- }
- }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Nov 15, 2006
+ *
*/
package com.bigdata.btree;
@@ -92,12 +93,6 @@
* on hand so that we can set this field.
*/
final transient protected AbstractBTree btree;
-
- /**
- * Can be used in debugging, testing or high integrity live scenarios to check
- * for concurrent writes triggered by evictions. This is currently disabled.
- */
- // volatile transient Throwable writing = null;
/**
* The parent of this node. This is null for the root node. The parent is
@@ -543,11 +538,8 @@
*/
parent = (Node) parent.copyOnWrite(oldId);
- assert !parent.isPersistent();
- } else {
- assert !parent.isPersistent();
}
-
+
/*
* Replace the reference to this child with the reference to the
* new child. This makes the old child inaccessible via
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,13 +23,13 @@
*/
/*
* Created on Nov 15, 2006
+ *
*/
package com.bigdata.btree;
import java.lang.ref.WeakReference;
import java.lang.reflect.Constructor;
import java.util.Iterator;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
@@ -900,24 +900,23 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/343
* @see https://sourceforge.net/apps/trac/bigdata/ticket/440
*/
- final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
- //final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
- //lock.lock();
- try {
- //synchronized(this) {
- if (/* autoCommit && */needsCheckpoint()) {
-
- /*
- * Flush the btree, write a checkpoint record, and return the
- * address of that checkpoint record. The [checkpoint] reference
- * is also updated.
- */
-
- return _writeCheckpoint2();
-
- }
- //}
+// final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
+ final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
+ lock.lock();
+ try {
+ if (/* autoCommit && */needsCheckpoint()) {
+
+ /*
+ * Flush the btree, write a checkpoint record, and return the
+ * address of that checkpoint record. The [checkpoint] reference
+ * is also updated.
+ */
+
+ return _writeCheckpoint2();
+
+ }
+
/*
* There have not been any writes on this btree or auto-commit is
* disabled.
@@ -1111,14 +1110,14 @@
@Override
final public long getRecordVersion() {
- return recordVersion;
+ return recordVersion;
}
@Override
final public long getMetadataAddr() {
- return metadata.getMetadataAddr();
+ return metadata.getMetadataAddr();
}
@@ -1314,7 +1313,7 @@
@Override
public long handleCommit(final long commitTime) {
- return writeCheckpoint2().getCheckpointAddr();
+ return writeCheckpoint2().getCheckpointAddr();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Nov 17, 2006
+ *
*/
package com.bigdata.btree;
@@ -60,78 +61,88 @@
}
final AbstractBTree btree = node.btree;
-
- if (btree.error != null) {
- throw new IllegalStateException("BTree is in an error state", btree.error);
- }
-
- try {
- // Note: This assert can be violated for a read-only B+Tree since
- // there is less synchronization.
- assert btree.isReadOnly()
- || btree.ndistinctOnWriteRetentionQueue > 0;
- btree.ndistinctOnWriteRetentionQueue--;
+ // Note: This assert can be violated for a read-only B+Tree since there
+ // is less synchronization.
+ assert btree.isReadOnly() || btree.ndistinctOnWriteRetentionQueue > 0;
- if (node.deleted) {
+ btree.ndistinctOnWriteRetentionQueue--;
+
+ if (node.deleted) {
- /*
- * Deleted nodes are ignored as they are evicted from the queue.
- */
+ /*
+ * Deleted nodes are ignored as they are evicted from the queue.
+ */
- return;
+ return;
- }
+ }
- // this does not permit transient nodes to be coded.
- if (node.dirty && btree.store != null) {
- // // this causes transient nodes to be coded on eviction.
- // if (node.dirty) {
+ // this does not permit transient nodes to be coded.
+ if (node.dirty && btree.store != null) {
+// // this causes transient nodes to be coded on eviction.
+// if (node.dirty) {
+
+ if (node.isLeaf()) {
- if (node.isLeaf()) {
+ /*
+ * A leaf is written out directly.
+ */
+
+ btree.writeNodeOrLeaf(node);
- /*
- * A leaf is written out directly.
- */
+ } else {
- btree.writeNodeOrLeaf(node);
+ /*
+ * A non-leaf node must be written out using a post-order
+ * traversal so that all dirty children are written through
+ * before the dirty parent. This is required in order to
+ * assign persistent identifiers to the dirty children.
+ */
- } else {
+ btree.writeNodeRecursive(node);
- /*
- * A non-leaf node must be written out using a post-order
- * traversal so that all dirty children are written through
- * before the dirty parent. This is required in order to
- * assign persistent identifiers to the dirty children.
- */
+ }
- btree.writeNodeRecursive(node);
+ // is a coded data record.
+ assert node.isCoded();
+
+ // no longer dirty.
+ assert !node.dirty;
+
+ if (btree.store != null) {
+
+ // object is persistent (has assigned addr).
+ assert ref.identity != PO.NULL;
+
+ }
+
+ } // isDirty
- }
+ // This does not insert into the cache. That is handled by writeNodeOrLeaf.
+// if (btree.globalLRU != null) {
+//
+// /*
+// * Add the INodeData or ILeafData object to the global LRU, NOT the
+// * Node or Leaf.
+// *
+// * Note: The global LRU touch only occurs on eviction from the write
+// * retention queue. This is nice because it limits the touches on
+// * the global LRU, which could otherwise be a hot spot. We do a
+// * touch whether or not the node was persisted since we are likely
+// * to return to the node in either case.
+// */
+//
+// final IAbstractNodeData delegate = node.getDelegate();
+//
+// assert delegate != null : node.toString();
+//
+// assert delegate.isCoded() : node.toString();
+//
+// btree.globalLRU.add(delegate);
+//
+// }
- // is a coded data record.
- assert node.isCoded();
-
- // no longer dirty.
- assert !node.dirty;
-
- if (btree.store != null) {
-
- // object is persistent (has assigned addr).
- assert ref.identity != PO.NULL;
-
- }
-
- } // isDirty
- } catch (Throwable e) {
- // If the btree is mutable that we are trying to evict a node then
- // mark it in error
- if (!btree.readOnly)
- btree.error = e;
-
- throw new Error(e);
- }
-
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Jan 10, 2008
+ *
*/
package com.bigdata.btree;
@@ -30,7 +31,6 @@
import java.util.Iterator;
import java.util.WeakHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -160,7 +160,7 @@
}
- // writeLock.lock();
+// writeLock.lock();
if(!writeLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -190,6 +190,8 @@
*/
public Lock readLock() {
+ final Lock readLock = readWriteLock.readLock();
+
try {
if(log.isDebugEnabled()) {
@@ -200,7 +202,7 @@
}
- // readLock.lock();
+// readLock.lock();
if(!readLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -213,73 +215,12 @@
throw new RuntimeException(ex);
}
-
- // We have attained the read lock
return readLock;
}
/**
- * WrappedLock is used to intercept lock/unlock calls to the readLock to
- * trigger calls to the AbstractBTree thread lock management that can be
- * used to identify whether the readlock is held by the current thread.
- * <p>
- * This is tested in AbstractBTree.touch() to determine whether the touch
- * should be ignored or trigger potential evictions.
- */
- class WrappedLock implements Lock {
-
- final Lock delegate;
-
- WrappedLock(Lock delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public void lock() {
- delegate.lock();
-
- ndx.readLockedThread();
- }
-
- @Override
- public void lockInterruptibly() throws InterruptedException {
- delegate.lockInterruptibly();
- }
-
- @Override
- public boolean tryLock() {
- final boolean ret = delegate.tryLock();
- if (ret) {
- ndx.readLockedThread();
- }
- return ret;
- }
-
- @Override
- public boolean tryLock(long time, TimeUnit unit)
- throws InterruptedException {
- final boolean ret = delegate.tryLock(time, unit);
- if (ret) {
- ndx.readLockedThread();
- }
- return ret;
- }
-
- @Override
- public void unlock() {
- delegate.unlock();
- ndx.readUnlockedThread();
- }
-
- @Override
- public Condition newCondition() {
- return delegate.newCondition();
- }
- }
-
- /**
* Acquire an appropriate lock depending on whether or not the procedure
* asserts that it is read-only.
*
@@ -328,9 +269,7 @@
*/
final private ReadWriteLock readWriteLock;
- final private Lock readLock;
-
- /**
+ /**
* Canonicalizing mapping for the locks used to control access to the
* unisolated index.
*/
@@ -411,8 +350,6 @@
this.readWriteLock = getReadWriteLock(ndx);
- this.readLock = new WrappedLock(readWriteLock.readLock());
-
}
/**
@@ -446,8 +383,8 @@
return readWriteLock;
}
- }
-
+ }
+
@Override
public String toString() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-09-02 13:45:16
|
Revision: 8633
http://sourceforge.net/p/bigdata/code/8633
Author: thompsonbry
Date: 2014-09-02 13:45:09 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Removed log @ WARN.
@see #1004 (Concurrent binding problem)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java 2014-09-02 13:34:48 UTC (rev 8632)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java 2014-09-02 13:45:09 UTC (rev 8633)
@@ -9,8 +9,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.util.StackInfoReport;
-
/**
* A low-contention/high concurrency weak value cache. This class can offer
* substantially less lock contention and hence greater performance than the
@@ -28,7 +26,6 @@
* {@link ConcurrentHashMap} which is a relatively expensive operation.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
* @param <K>
* The generic type of the keys.
* @param <V>
@@ -473,16 +470,9 @@
final WeakReference<V> oldRef = map.putIfAbsent(k, ref);
final V oldVal = oldRef == null ? null : oldRef.get();
-
+
if (oldRef != null && oldVal == null) {
- // Potential race, let's make it more likely
-// try {
-// Thread.sleep(50);
-// } catch (InterruptedException e) {
-// throw new RuntimeException("Unexpected?", e);
-// }
-
/*
* There was an entry under the key but its reference has been
* cleared. A cleared value paired to the key is equivalent to
@@ -494,7 +484,7 @@
*/
if (map.replace(k, oldRef, ref)) {
-
+
if (queue != null) {
// no reference under that key.
@@ -517,11 +507,18 @@
// the old value for the key was a cleared reference.
return null;
- } else { // lost potential concurrent race, so make recursive call to ensure correct value is returned
- log.warn("REMOVE: Lost Race... try again", new StackInfoReport());
-
- // throw new IllegalStateException("REMOVE: WTF?");
- return putIfAbsent(k, v);
+ } else {
+
+ /**
+ * We lost a potential concurrent data race, so make
+ * recursive call to ensure correct value is returned.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/1004">
+ * Concurrent binding problem </a>
+ */
+
+ return putIfAbsent(k, v);
+
}
}
@@ -779,7 +776,6 @@
* Adds the key to the weak reference.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
* @param <K>
* @param <V>
*/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-09-02 13:35:02
|
Revision: 8632
http://sourceforge.net/p/bigdata/code/8632
Author: martyncutcher
Date: 2014-09-02 13:34:48 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Provide protection against eviction of Mutable BTree nodes during read-only operation #855. Achieved by checking for read-lock held on current thread and avoiding a touch that could cause eviction.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -36,9 +36,14 @@
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.FutureTask;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -87,6 +92,7 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Split;
import com.bigdata.util.InnerCause;
+import com.bigdata.util.StackInfoReport;
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
@@ -249,6 +255,8 @@
* mutation.
*/
final protected boolean readOnly;
+
+ final ReentrantReadWriteLock lock;
/**
* Optional cache for {@link INodeData} and {@link ILeafData} instances and
@@ -959,6 +967,8 @@
this.store = store;
this.readOnly = readOnly;
+
+ this.lock = UnisolatedReadWriteIndex.getReadWriteLock(this);
// /*
// * The Memoizer is not used by the mutable B+Tree since it is not safe
@@ -1976,6 +1986,8 @@
}
};
+
+ volatile Throwable error;
final public Object insert(Object key, Object value) {
@@ -3368,7 +3380,14 @@
}
- doSyncTouch(node);
+ // If a mutable tree and a read-only operation then ensure there are no evictions from this thread
+ final Integer rcount = threadLockMap.get(Thread.currentThread().getId());
+ if (!isReadOnly() && rcount != null /* && false this should stochastically fail!*/) {
+ // NOP
+ assert rcount.intValue() > 0;
+ } else {
+ doSyncTouch(node);
+ }
}
@@ -3432,7 +3451,7 @@
*/
// assert isReadOnly() || ndistinctOnWriteRetentionQueue > 0;
-
+
node.referenceCount++;
if (!writeRetentionQueue.add(node)) {
@@ -3597,6 +3616,15 @@
}
+ private void badNode(final AbstractNode<?> node) {
+// try {
+// Thread.sleep(50);
+// } catch (InterruptedException e) {
+// // ignore;
+// }
+ throw new AssertionError("ReadOnly and identity: " + node.identity);
+ }
+
/**
* Codes the node and writes the coded record on the store (non-recursive).
* The node MUST be dirty. If the node has a parent, then the parent is
@@ -3616,7 +3644,31 @@
* @return The persistent identity assigned by the store.
*/
protected long writeNodeOrLeaf(final AbstractNode<?> node) {
-
+
+ /*
+ * The check for the writeLock held on an update is currently disabled since
+ * a number of Unit tests failed with it active. The exception information
+ * indicates that the writeLock is not held in any of the test failure scenarios, so
+ * the thread check does not fail due to submitted tasks.
+ * TODO: we need to clarify the requirements for holding the writeLock across any update
+ */
+ if (false && !lock.isWriteLockedByCurrentThread()) {
+ throw new AssertionError("WriteLock not held, readOnly: " + this.isReadOnly() + ", locked by other: " + lock.isWriteLocked());
+ }
+
+ if (error != null) {
+ throw new IllegalStateException("BTree is in an error state", error);
+ }
+
+// if (node.writing != null) {
+// final Throwable remote = node.writing;
+// node.writing = null;
+//
+// throw new AssertionError("Duplicate Call - ", remote);
+// }
+//
+ // node.writing = new StackInfoReport("Write Guard");
+
assert root != null; // i.e., isOpen().
assert node != null;
assert node.btree == this;
@@ -3640,7 +3692,10 @@
* TestMROWTransactions might also demonstrate an issue
* occasionally. If so, then check for the same root cause.
*/
- assert !node.isReadOnly();
+ if (node.isReadOnly()) {
+ badNode(node); // supports debugging
+ }
+ assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
assertNotReadOnly();
/*
@@ -3740,6 +3795,14 @@
// No longer dirty (prevents re-coding on re-eviction).
node.setDirty(false);
+// if (node.writing == null) {
+// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
+//
+// throw new AssertionError("Concurrent modification of thread guard");
+// }
+
+// node.writing = null;
+
return 0L;
}
@@ -3829,6 +3892,14 @@
}
+// if (node.writing == null) {
+// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
+//
+// throw new AssertionError("Concurrent modification of thread guard");
+// }
+//
+// node.writing = null;
+
return addr;
}
@@ -3855,40 +3926,6 @@
if (addr == IRawStore.NULL)
throw new IllegalArgumentException();
-// final Long addr2 = Long.valueOf(addr);
-//
-// if (storeCache != null) {
-//
-// // test cache : will touch global LRU iff found.
-// final IAbstractNodeData data = (IAbstractNodeData) storeCache
-// .get(addr);
-//
-// if (data != null) {
-//
-// // Node and Leaf MUST NOT make it into the global LRU or store
-// // cache!
-// assert !(data instanceof AbstractNode<?>);
-//
-// final AbstractNode<?> node;
-//
-// if (data.isLeaf()) {
-//
-// node = nodeSer.nodeFactory.allocLeaf(this, addr,
-// (ILeafData) data);
-//
-// } else {
-//
-// node = nodeSer.nodeFactory.allocNode(this, addr,
-// (INodeData) data);
-//
-// }
-//
-// // cache hit.
-// return node;
-//
-// }
-//
-// }
final ByteBuffer tmp;
{
@@ -3945,21 +3982,6 @@
}
-// if (storeCache != null) {
-//
-// // update cache : will touch global LRU iff cache is modified.
-// final IAbstractNodeData data2 = (IAbstractNodeData) storeCache
-// .putIfAbsent(addr2, data);
-//
-// if (data2 != null) {
-//
-// // concurrent insert, use winner's value.
-// data = data2;
-//
-// }
-//
-// }
-
// wrap as Node or Leaf.
final AbstractNode<?> node = nodeSer.wrap(this, addr, data);
@@ -4303,4 +4325,35 @@
}
+
+ /**
+ * Maintain count of readLocks on this Thread. Use ThreadLocal for now, but
+ * could switch to HashMap if necessary
+ */
+// ThreadLocal<AtomicInteger> threadReadLockCount = new ThreadLocal<AtomicInteger>();
+ ConcurrentHashMap<Long, Integer> threadLockMap = new ConcurrentHashMap<Long, Integer>();
+
+ public void readLockedThread() {
+// final AtomicInteger ai = threadReadLockCount.get();
+// if (ai == null) {
+// threadReadLockCount.set(new AtomicInteger(1));
+// } else {
+// ai.incrementAndGet();
+// }
+ final long thisThreadId = Thread.currentThread().getId();
+ final Integer entry = threadLockMap.get(thisThreadId);
+ final Integer newVal = entry == null ? 1 : 1 + entry.intValue();
+ threadLockMap.put(thisThreadId, newVal);
+ }
+
+ public void readUnlockedThread() {
+ final long thisThreadId = Thread.currentThread().getId();
+ final Integer entry = threadLockMap.get(thisThreadId);
+ assert entry != null;
+ if (entry.intValue() == 1) {
+ threadLockMap.remove(thisThreadId);
+ } else {
+ threadLockMap.put(thisThreadId, entry.intValue() - 1);
+ }
+ }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -92,6 +92,12 @@
* on hand so that we can set this field.
*/
final transient protected AbstractBTree btree;
+
+ /**
+ * Can be used in debugging, testing or high integrity live scenarios to check
+ * for concurrent writes triggered by evictions. This is currently disabled.
+ */
+ // volatile transient Throwable writing = null;
/**
* The parent of this node. This is null for the root node. The parent is
@@ -537,8 +543,11 @@
*/
parent = (Node) parent.copyOnWrite(oldId);
+ assert !parent.isPersistent();
+ } else {
+ assert !parent.isPersistent();
}
-
+
/*
* Replace the reference to this child with the reference to the
* new child. This makes the old child inaccessible via
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -29,6 +29,7 @@
import java.lang.ref.WeakReference;
import java.lang.reflect.Constructor;
import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
@@ -899,23 +900,24 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/343
* @see https://sourceforge.net/apps/trac/bigdata/ticket/440
*/
-// final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
- final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
- lock.lock();
- try {
+ final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
+ //final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
+ //lock.lock();
+ try {
+ //synchronized(this) {
+ if (/* autoCommit && */needsCheckpoint()) {
+
+ /*
+ * Flush the btree, write a checkpoint record, and return the
+ * address of that checkpoint record. The [checkpoint] reference
+ * is also updated.
+ */
+
+ return _writeCheckpoint2();
+
+ }
+ //}
- if (/* autoCommit && */needsCheckpoint()) {
-
- /*
- * Flush the btree, write a checkpoint record, and return the
- * address of that checkpoint record. The [checkpoint] reference
- * is also updated.
- */
-
- return _writeCheckpoint2();
-
- }
-
/*
* There have not been any writes on this btree or auto-commit is
* disabled.
@@ -1109,14 +1111,14 @@
@Override
final public long getRecordVersion() {
- return recordVersion;
+ return recordVersion;
}
@Override
final public long getMetadataAddr() {
- return metadata.getMetadataAddr();
+ return metadata.getMetadataAddr();
}
@@ -1312,7 +1314,7 @@
@Override
public long handleCommit(final long commitTime) {
- return writeCheckpoint2().getCheckpointAddr();
+ return writeCheckpoint2().getCheckpointAddr();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -60,88 +60,78 @@
}
final AbstractBTree btree = node.btree;
+
+ if (btree.error != null) {
+ throw new IllegalStateException("BTree is in an error state", btree.error);
+ }
+
+ try {
+ // Note: This assert can be violated for a read-only B+Tree since
+ // there is less synchronization.
+ assert btree.isReadOnly()
+ || btree.ndistinctOnWriteRetentionQueue > 0;
- // Note: This assert can be violated for a read-only B+Tree since there
- // is less synchronization.
- assert btree.isReadOnly() || btree.ndistinctOnWriteRetentionQueue > 0;
+ btree.ndistinctOnWriteRetentionQueue--;
- btree.ndistinctOnWriteRetentionQueue--;
-
- if (node.deleted) {
+ if (node.deleted) {
- /*
- * Deleted nodes are ignored as they are evicted from the queue.
- */
+ /*
+ * Deleted nodes are ignored as they are evicted from the queue.
+ */
- return;
+ return;
- }
+ }
- // this does not permit transient nodes to be coded.
- if (node.dirty && btree.store != null) {
-// // this causes transient nodes to be coded on eviction.
-// if (node.dirty) {
-
- if (node.isLeaf()) {
+ // this does not permit transient nodes to be coded.
+ if (node.dirty && btree.store != null) {
+ // // this causes transient nodes to be coded on eviction.
+ // if (node.dirty) {
- /*
- * A leaf is written out directly.
- */
-
- btree.writeNodeOrLeaf(node);
+ if (node.isLeaf()) {
- } else {
+ /*
+ * A leaf is written out directly.
+ */
- /*
- * A non-leaf node must be written out using a post-order
- * traversal so that all dirty children are written through
- * before the dirty parent. This is required in order to
- * assign persistent identifiers to the dirty children.
- */
+ btree.writeNodeOrLeaf(node);
- btree.writeNodeRecursive(node);
+ } else {
- }
+ /*
+ * A non-leaf node must be written out using a post-order
+ * traversal so that all dirty children are written through
+ * before the dirty parent. This is required in order to
+ * assign persistent identifiers to the dirty children.
+ */
- // is a coded data record.
- assert node.isCoded();
-
- // no longer dirty.
- assert !node.dirty;
-
- if (btree.store != null) {
-
- // object is persistent (has assigned addr).
- assert ref.identity != PO.NULL;
-
- }
-
- } // isDirty
+ btree.writeNodeRecursive(node);
- // This does not insert into the cache. That is handled by writeNodeOrLeaf.
-// if (btree.globalLRU != null) {
-//
-// /*
-// * Add the INodeData or ILeafData object to the global LRU, NOT the
-// * Node or Leaf.
-// *
-// * Note: The global LRU touch only occurs on eviction from the write
-// * retention queue. This is nice because it limits the touches on
-// * the global LRU, which could otherwise be a hot spot. We do a
-// * touch whether or not the node was persisted since we are likely
-// * to return to the node in either case.
-// */
-//
-// final IAbstractNodeData delegate = node.getDelegate();
-//
-// assert delegate != null : node.toString();
-//
-// assert delegate.isCoded() : node.toString();
-//
-// btree.globalLRU.add(delegate);
-//
-// }
+ }
+ // is a coded data record.
+ assert node.isCoded();
+
+ // no longer dirty.
+ assert !node.dirty;
+
+ if (btree.store != null) {
+
+ // object is persistent (has assigned addr).
+ assert ref.identity != PO.NULL;
+
+ }
+
+ } // isDirty
+ } catch (Throwable e) {
+ // If the btree is mutable that we are trying to evict a node then
+ // mark it in error
+ if (!btree.readOnly)
+ btree.error = e;
+
+ throw new Error(e);
+ }
+
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -30,6 +30,7 @@
import java.util.Iterator;
import java.util.WeakHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -159,7 +160,7 @@
}
-// writeLock.lock();
+ // writeLock.lock();
if(!writeLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -189,8 +190,6 @@
*/
public Lock readLock() {
- final Lock readLock = readWriteLock.readLock();
-
try {
if(log.isDebugEnabled()) {
@@ -201,7 +200,7 @@
}
-// readLock.lock();
+ // readLock.lock();
if(!readLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -214,12 +213,73 @@
throw new RuntimeException(ex);
}
+
+ // We have attained the read lock
return readLock;
}
/**
+ * WrappedLock is used to intercept lock/unlock calls to the readLock to
+ * trigger calls to the AbstractBTree thread lock management that can be
+ * used to identify whether the readlock is held by the current thread.
+ * <p>
+ * This is tested in AbstractBTree.touch() to determine whether the touch
+ * should be ignored or trigger potential evictions.
+ */
+ class WrappedLock implements Lock {
+
+ final Lock delegate;
+
+ WrappedLock(Lock delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void lock() {
+ delegate.lock();
+
+ ndx.readLockedThread();
+ }
+
+ @Override
+ public void lockInterruptibly() throws InterruptedException {
+ delegate.lockInterruptibly();
+ }
+
+ @Override
+ public boolean tryLock() {
+ final boolean ret = delegate.tryLock();
+ if (ret) {
+ ndx.readLockedThread();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean tryLock(long time, TimeUnit unit)
+ throws InterruptedException {
+ final boolean ret = delegate.tryLock(time, unit);
+ if (ret) {
+ ndx.readLockedThread();
+ }
+ return ret;
+ }
+
+ @Override
+ public void unlock() {
+ delegate.unlock();
+ ndx.readUnlockedThread();
+ }
+
+ @Override
+ public Condition newCondition() {
+ return delegate.newCondition();
+ }
+ }
+
+ /**
* Acquire an appropriate lock depending on whether or not the procedure
* asserts that it is read-only.
*
@@ -268,7 +328,9 @@
*/
final private ReadWriteLock readWriteLock;
- /**
+ final private Lock readLock;
+
+ /**
* Canonicalizing mapping for the locks used to control access to the
* unisolated index.
*/
@@ -349,6 +411,8 @@
this.readWriteLock = getReadWriteLock(ndx);
+ this.readLock = new WrappedLock(readWriteLock.readLock());
+
}
/**
@@ -382,8 +446,8 @@
return readWriteLock;
}
- }
-
+ }
+
@Override
public String toString() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-09-02 12:48:25
|
Revision: 8631
http://sourceforge.net/p/bigdata/code/8631
Author: martyncutcher
Date: 2014-09-02 12:48:16 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Fix to ConcurrentWeakValueCache that could result in an error in the event of a data race in putIfAbsent - ticket #1004
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java 2014-08-28 21:18:14 UTC (rev 8630)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/cache/ConcurrentWeakValueCache.java 2014-09-02 12:48:16 UTC (rev 8631)
@@ -9,6 +9,8 @@
import org.apache.log4j.Logger;
+import com.bigdata.util.StackInfoReport;
+
/**
* A low-contention/high concurrency weak value cache. This class can offer
* substantially less lock contention and hence greater performance than the
@@ -471,9 +473,16 @@
final WeakReference<V> oldRef = map.putIfAbsent(k, ref);
final V oldVal = oldRef == null ? null : oldRef.get();
-
+
if (oldRef != null && oldVal == null) {
+ // Potential race, let's make it more likely
+// try {
+// Thread.sleep(50);
+// } catch (InterruptedException e) {
+// throw new RuntimeException("Unexpected?", e);
+// }
+
/*
* There was an entry under the key but its reference has been
* cleared. A cleared value paired to the key is equivalent to
@@ -485,7 +494,7 @@
*/
if (map.replace(k, oldRef, ref)) {
-
+
if (queue != null) {
// no reference under that key.
@@ -508,6 +517,11 @@
// the old value for the key was a cleared reference.
return null;
+ } else { // lost potential concurrent race, so make recursive call to ensure correct value is returned
+ log.warn("REMOVE: Lost Race... try again", new StackInfoReport());
+
+ // throw new IllegalStateException("REMOVE: WTF?");
+ return putIfAbsent(k, v);
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <jer...@us...> - 2014-08-28 21:18:16
|
Revision: 8630
http://sourceforge.net/p/bigdata/code/8630
Author: jeremy_carroll
Date: 2014-08-28 21:18:14 +0000 (Thu, 28 Aug 2014)
Log Message:
-----------
Consequential changes for trac-765, given that MemorySortOp has been redefined to not discard solutions
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestMemorySortOp.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestMemorySortOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestMemorySortOp.java 2014-08-28 21:18:02 UTC (rev 8629)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestMemorySortOp.java 2014-08-28 21:18:14 UTC (rev 8630)
@@ -335,6 +335,8 @@
* solutions in which those variables are not bound will cause type errors.
* Unless the value expressions are evaluated before we sort the solutions
* those type errors will propagate out of the sort and fail the query.
+ * Correct treatment is to treat the type errors as unbound variables
+ * see trac-765
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testComputedValueExpressions()
@@ -351,7 +353,8 @@
final ISortOrder<?> sors[] = new ISortOrder[] { //
new SortOrder(new Bind(z,new MathBOp(x, y, MathBOp.MathOp.PLUS,new GlobalAnnotations(getName(), ITx.READ_COMMITTED))), false/* asc */),//
- new SortOrder(y, false/* asc */) //
+ new SortOrder(y, false/* asc */), //
+ new SortOrder(x, true/* asc */), //
};
final int sortOpId = 1;
@@ -390,16 +393,16 @@
//
final IBindingSet expected [] = new IBindingSet []
{
-// new ListBindingSet ( new IVariable<?> [] { y }, new IConstant [] { _1 } ) // dropped by type error.
-// , new ListBindingSet ( new IVariable<?> [] {}, new IConstant [] {} ) // dropped by type error.
-// , new ListBindingSet ( new IVariable<?> [] { x }, new IConstant [] { _3 } ) // dropped by type error.
- new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _1, _5 } )
+ new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _1, _5 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _2, _4 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _4, _2 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _4, _1 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _1, _3 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _2, _2 } )
, new ListBindingSet ( new IVariable<?> [] { x, y }, new IConstant [] { _1, _1 } )
+ , new ListBindingSet ( new IVariable<?> [] { y }, new IConstant [] { _1 } ) // type error.
+ , new ListBindingSet ( new IVariable<?> [] {}, new IConstant [] {} ) // type error.
+ , new ListBindingSet ( new IVariable<?> [] { x }, new IConstant [] { _3 } ) // type error.
} ;
final BOpStats stats = query.newStats () ;
@@ -441,7 +444,7 @@
assertEquals ( 1, stats.chunksIn.get () ) ;
assertEquals ( 10, stats.unitsIn.get () ) ;
- assertEquals ( 7, stats.unitsOut.get () ) ;
+ assertEquals ( 10, stats.unitsOut.get () ) ;
assertEquals(1, stats.chunksOut.get());
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <jer...@us...> - 2014-08-28 21:17:56
|
Revision: 8628
http://sourceforge.net/p/bigdata/code/8628
Author: jeremy_carroll
Date: 2014-08-28 21:17:49 +0000 (Thu, 28 Aug 2014)
Log Message:
-----------
New test for trac-765 ORDER BY invalid expression
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestTickets.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.rq
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.srx
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.trig
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestTickets.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestTickets.java 2014-08-28 16:12:03 UTC (rev 8627)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestTickets.java 2014-08-28 21:17:49 UTC (rev 8628)
@@ -334,4 +334,14 @@
}
+ public void test_ticket_765() throws Exception {
+ new TestHelper("ticket-765",// testURI,
+ "ticket-765.rq",// queryFileURL
+ "ticket-765.trig",// dataFileURL
+ "ticket-765.srx",// resultFileURL
+ false // checkOrder (because only one solution)
+ ).runTest();
+
+ }
+
}
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.rq 2014-08-28 21:17:49 UTC (rev 8628)
@@ -0,0 +1,5 @@
+SELECT ?bound ?unbound
+{
+ BIND ( "somevalue" as ?bound )
+}
+ORDER BY LCASE(?unbound)
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.srx 2014-08-28 21:17:49 UTC (rev 8628)
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="bound"/>
+ <variable name="unbound"/>
+ </head>
+ <results>
+ <result>
+ <binding name="bound">
+ <literal>somevalue</literal>
+ </binding>
+ </result>
+ </results>
+</sparql>
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/ticket-765.trig
===================================================================
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-08-28 16:12:07
|
Revision: 8627
http://sourceforge.net/p/bigdata/code/8627
Author: mrpersonick
Date: 2014-08-28 16:12:03 +0000 (Thu, 28 Aug 2014)
Log Message:
-----------
Ticket #714: Sesame 2.7. Added BindingsClause to StaticAnalysis getDefinitely/MaybeProducedBindings. Made BindingsClause an IJoinNode.
Modified Paths:
--------------
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-08-27 22:50:52 UTC (rev 8626)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2014-08-28 16:12:03 UTC (rev 8627)
@@ -27,6 +27,7 @@
package com.bigdata.rdf.sparql.ast;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -35,6 +36,7 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IVariable;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode.Annotations;
/**
* The solutions declared by a BINDINGS clause.
@@ -43,14 +45,15 @@
* @version $Id$
*/
public class BindingsClause extends GroupMemberNodeBase<BindingsClause>
- implements IBindingProducerNode {
+ implements IBindingProducerNode, IJoinNode {
/**
*
*/
private static final long serialVersionUID = 1L;
- public interface Annotations extends ASTBase.Annotations {
+ public interface Annotations extends ASTBase.Annotations,
+ IJoinNode.Annotations {
/**
* The ordered set of declared variables for which there MIGHT be a
@@ -227,4 +230,37 @@
}
+ @Override
+ final public List<FilterNode> getAttachedJoinFilters() {
+
+ @SuppressWarnings("unchecked")
+ final List<FilterNode> filters = (List<FilterNode>) getProperty(Annotations.FILTERS);
+
+ if (filters == null) {
+
+ return Collections.emptyList();
+
+ }
+
+ return Collections.unmodifiableList(filters);
+
+ }
+
+ @Override
+ final public void setAttachedJoinFilters(final List<FilterNode> filters) {
+
+ setProperty(Annotations.FILTERS, filters);
+
+ }
+
+ @Override
+ public boolean isOptional() {
+ return false;
+ }
+
+ @Override
+ public boolean isMinus() {
+ return false;
+ }
+
}
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-08-27 22:50:52 UTC (rev 8626)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2014-08-28 16:12:03 UTC (rev 8627)
@@ -912,8 +912,14 @@
// NOP.
+ } else if(node instanceof BindingsClause) {
+
+ final BindingsClause bc = (BindingsClause) node;
+
+ vars.addAll(bc.getDeclaredVariables());
+
} else {
-
+
throw new AssertionError(node.toString());
}
@@ -1180,8 +1186,10 @@
} else if(child instanceof BindingsClause) {
- // NOP
+ final BindingsClause bc = (BindingsClause) child;
+ vars.addAll(bc.getDeclaredVariables());
+
} else {
throw new AssertionError(child.toString());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-08-27 22:50:55
|
Revision: 8626
http://sourceforge.net/p/bigdata/code/8626
Author: tobycraig
Date: 2014-08-27 22:50:52 +0000 (Wed, 27 Aug 2014)
Log Message:
-----------
Fixed tab switching shortcut moving two tabs when an editor is focused
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-27 21:49:55 UTC (rev 8625)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-27 22:50:52 UTC (rev 8626)
@@ -14,7 +14,6 @@
lineNumbers: true,
matchBrackets: true,
mode: 'sparql',
- extraKeys: {'Ctrl-,': moveTabLeft, 'Ctrl-.': moveTabRight}
};
// key is value of RDF type selector, value is name of CodeMirror mode
var RDF_MODES = {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-08-27 21:49:59
|
Revision: 8625
http://sourceforge.net/p/bigdata/code/8625
Author: tobycraig
Date: 2014-08-27 21:49:55 +0000 (Wed, 27 Aug 2014)
Log Message:
-----------
Added documentation links above query/update editors
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-08-27 21:08:13 UTC (rev 8624)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-08-27 21:49:55 UTC (rev 8625)
@@ -193,6 +193,10 @@
clear: both;
}
+.links {
+ float: left;
+}
+
.namespace-shortcuts {
float: right;
margin-bottom: 20px;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-08-27 21:08:13 UTC (rev 8624)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-08-27 21:49:55 UTC (rev 8625)
@@ -35,6 +35,11 @@
<div class="box">
+ <div class="links">
+ <a href="http://wiki.bigdata.com/wiki/index.php/Main_Page">Wiki</a> -
+ <a href="http://www.w3.org/TR/sparql11-update/">SPARQL Update</a>
+ </div>
+
<div class="namespace-shortcuts">
</div>
@@ -87,6 +92,11 @@
<div class="tab" id="query-tab">
<div class="box">
+ <div class="links">
+ <a href="http://wiki.bigdata.com/wiki/index.php/Main_Page">Wiki</a> -
+ <a href="http://www.w3.org/TR/sparql11-query/">SPARQL Query</a>
+ </div>
+
<div class="namespace-shortcuts">
</div>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-08-27 21:08:22
|
Revision: 8624
http://sourceforge.net/p/bigdata/code/8624
Author: tobycraig
Date: 2014-08-27 21:08:13 +0000 (Wed, 27 Aug 2014)
Log Message:
-----------
Added bracket matching in query editors
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/vendor/cm-addons/matchbrackets.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-08-25 16:18:12 UTC (rev 8623)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-08-27 21:08:13 UTC (rev 8624)
@@ -294,6 +294,7 @@
<script>window.jQuery || document.write('<script src="/bigdata/html/js/vendor/jquery.min.js"><\/script>')</script>
<script src="/bigdata/html/js/vendor/jquery.hotkeys.js"></script>
<script src="/bigdata/html/js/vendor/codemirror.js"></script>
+ <script src="/bigdata/html/js/vendor/cm-addons/matchbrackets.js"></script>
<script src="/bigdata/html/js/vendor/cm-addons/placeholder.js"></script>
<script src="/bigdata/html/js/vendor/cm-modes/javascript.js"></script>
<script src="/bigdata/html/js/vendor/cm-modes/ntriples.js"></script>
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/vendor/cm-addons/matchbrackets.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/vendor/cm-addons/matchbrackets.js (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/vendor/cm-addons/matchbrackets.js 2014-08-27 21:08:13 UTC (rev 8624)
@@ -0,0 +1,120 @@
+// CodeMirror, copyright (c) by Marijn Haverbeke and others
+// Distributed under an MIT license: http://codemirror.net/LICENSE
+
+(function(mod) {
+ if (typeof exports == "object" && typeof module == "object") // CommonJS
+ mod(require("../../lib/codemirror"));
+ else if (typeof define == "function" && define.amd) // AMD
+ define(["../../lib/codemirror"], mod);
+ else // Plain browser env
+ mod(CodeMirror);
+})(function(CodeMirror) {
+ var ie_lt8 = /MSIE \d/.test(navigator.userAgent) &&
+ (document.documentMode == null || document.documentMode < 8);
+
+ var Pos = CodeMirror.Pos;
+
+ var matching = {"(": ")>", ")": "(<", "[": "]>", "]": "[<", "{": "}>", "}": "{<"};
+
+ function findMatchingBracket(cm, where, strict, config) {
+ var line = cm.getLineHandle(where.line), pos = where.ch - 1;
+ var match = (pos >= 0 && matching[line.text.charAt(pos)]) || matching[line.text.charAt(++pos)];
+ if (!match) return null;
+ var dir = match.charAt(1) == ">" ? 1 : -1;
+ if (strict && (dir > 0) != (pos == where.ch)) return null;
+ var style = cm.getTokenTypeAt(Pos(where.line, pos + 1));
+
+ var found = scanForBracket(cm, Pos(where.line, pos + (dir > 0 ? 1 : 0)), dir, style || null, config);
+ if (found == null) return null;
+ return {from: Pos(where.line, pos), to: found && found.pos,
+ match: found && found.ch == match.charAt(0), forward: dir > 0};
+ }
+
+ // bracketRegex is used to specify which type of bracket to scan
+ // should be a regexp, e.g. /[[\]]/
+ //
+ // Note: If "where" is on an open bracket, then this bracket is ignored.
+ //
+ // Returns false when no bracket was found, null when it reached
+ // maxScanLines and gave up
+ function scanForBracket(cm, where, dir, style, config) {
+ var maxScanLen = (config && config.maxScanLineLength) || 10000;
+ var maxScanLines = (config && config.maxScanLines) || 1000;
+
+ var stack = [];
+ var re = config && config.bracketRegex ? config.bracketRegex : /[(){}[\]]/;
+ var lineEnd = dir > 0 ? Math.min(where.line + maxScanLines, cm.lastLine() + 1)
+ : Math.max(cm.firstLine() - 1, where.line - maxScanLines);
+ for (var lineNo = where.line; lineNo != lineEnd; lineNo += dir) {
+ var line = cm.getLine(lineNo);
+ if (!line) continue;
+ var pos = dir > 0 ? 0 : line.length - 1, end = dir > 0 ? line.length : -1;
+ if (line.length > maxScanLen) continue;
+ if (lineNo == where.line) pos = where.ch - (dir < 0 ? 1 : 0);
+ for (; pos != end; pos += dir) {
+ var ch = line.charAt(pos);
+ if (re.test(ch) && (style === undefined || cm.getTokenTypeAt(Pos(lineNo, pos + 1)) == style)) {
+ var match = matching[ch];
+ if ((match.charAt(1) == ">") == (dir > 0)) stack.push(ch);
+ else if (!stack.length) return {pos: Pos(lineNo, pos), ch: ch};
+ else stack.pop();
+ }
+ }
+ }
+ return lineNo - dir == (dir > 0 ? cm.lastLine() : cm.firstLine()) ? false : null;
+ }
+
+ function matchBrackets(cm, autoclear, config) {
+ // Disable brace matching in long lines, since it'll cause hugely slow updates
+ var maxHighlightLen = cm.state.matchBrackets.maxHighlightLineLength || 1000;
+ var marks = [], ranges = cm.listSelections();
+ for (var i = 0; i < ranges.length; i++) {
+ var match = ranges[i].empty() && findMatchingBracket(cm, ranges[i].head, false, config);
+ if (match && cm.getLine(match.from.line).length <= maxHighlightLen) {
+ var style = match.match ? "CodeMirror-matchingbracket" : "CodeMirror-nonmatchingbracket";
+ marks.push(cm.markText(match.from, Pos(match.from.line, match.from.ch + 1), {className: style}));
+ if (match.to && cm.getLine(match.to.line).length <= maxHighlightLen)
+ marks.push(cm.markText(match.to, Pos(match.to.line, match.to.ch + 1), {className: style}));
+ }
+ }
+
+ if (marks.length) {
+ // Kludge to work around the IE bug from issue #1193, where text
+ // input stops going to the textare whever this fires.
+ if (ie_lt8 && cm.state.focused) cm.display.input.focus();
+
+ var clear = function() {
+ cm.operation(function() {
+ for (var i = 0; i < marks.length; i++) marks[i].clear();
+ });
+ };
+ if (autoclear) setTimeout(clear, 800);
+ else return clear;
+ }
+ }
+
+ var currentlyHighlighted = null;
+ function doMatchBrackets(cm) {
+ cm.operation(function() {
+ if (currentlyHighlighted) {currentlyHighlighted(); currentlyHighlighted = null;}
+ currentlyHighlighted = matchBrackets(cm, false, cm.state.matchBrackets);
+ });
+ }
+
+ CodeMirror.defineOption("matchBrackets", false, function(cm, val, old) {
+ if (old && old != CodeMirror.Init)
+ cm.off("cursorActivity", doMatchBrackets);
+ if (val) {
+ cm.state.matchBrackets = typeof val == "object" ? val : {};
+ cm.on("cursorActivity", doMatchBrackets);
+ }
+ });
+
+ CodeMirror.defineExtension("matchBrackets", function() {matchBrackets(this, true);});
+ CodeMirror.defineExtension("findMatchingBracket", function(pos, strict, config){
+ return findMatchingBracket(this, pos, strict, config);
+ });
+ CodeMirror.defineExtension("scanForBracket", function(pos, dir, style, config){
+ return scanForBracket(this, pos, dir, style, config);
+ });
+});
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-25 16:18:12 UTC (rev 8623)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-27 21:08:13 UTC (rev 8624)
@@ -12,6 +12,7 @@
var EDITORS = {}, ERROR_LINE_MARKERS = {}, ERROR_CHARACTER_MARKERS = {};
var CODEMIRROR_DEFAULTS = {
lineNumbers: true,
+ matchBrackets: true,
mode: 'sparql',
extraKeys: {'Ctrl-,': moveTabLeft, 'Ctrl-.': moveTabRight}
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-08-25 16:18:17
|
Revision: 8623
http://sourceforge.net/p/bigdata/code/8623
Author: mrpersonick
Date: 2014-08-25 16:18:12 +0000 (Mon, 25 Aug 2014)
Log Message:
-----------
actually per the spec, no, we don't want to run now() more than once
Modified Paths:
--------------
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-25 14:57:01 UTC (rev 8622)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-25 16:18:12 UTC (rev 8623)
@@ -4645,7 +4645,7 @@
}
- if (op instanceof UUIDBOp || op instanceof NowBOp) {
+ if (op instanceof UUIDBOp) {// || op instanceof NowBOp) {
/*
* We cannot pre-generate these, they need to be unique
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-08-25 14:57:08
|
Revision: 8622
http://sourceforge.net/p/bigdata/code/8622
Author: mrpersonick
Date: 2014-08-25 14:57:01 +0000 (Mon, 25 Aug 2014)
Log Message:
-----------
#714: attach values operator to pipeline
Modified Paths:
--------------
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-25 14:40:28 UTC (rev 8621)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-25 14:57:01 UTC (rev 8622)
@@ -1469,6 +1469,19 @@
}
/**
+ * TODO Grab the binding sets from the BindingsClause, attach them to
+ * the query as a named subquery with a hash index, and then add a
+ * named subquery include to the pipeline right here.
+ */
+ private static PipelineOp addValues(PipelineOp left,
+ final BindingsClause bc,
+ final Set<IVariable<?>> doneSet, final AST2BOpContext ctx) {
+
+ return left;
+
+ }
+
+ /**
* Return <code>true</code> if we can optimize this INCLUDE with a SCAN of
* the named solution set and a nested inner loop to join against left
* solutions from the pipeline having a known low cardinality.
@@ -2657,6 +2670,13 @@
left = addNamedSubqueryInclude(left,
(NamedSubqueryInclude) child, doneSet, ctx);
continue;
+ } else if (child instanceof BindingsClause) {
+ /*
+ * VALUES clause
+ */
+ left = addValues(left,
+ (BindingsClause) child, doneSet, ctx);
+ continue;
} else if (child instanceof SubqueryRoot) {
final SubqueryRoot subquery = (SubqueryRoot) child;
switch (subquery.getQueryType()) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-08-25 14:40:33
|
Revision: 8621
http://sourceforge.net/p/bigdata/code/8621
Author: mrpersonick
Date: 2014-08-25 14:40:28 +0000 (Mon, 25 Aug 2014)
Log Message:
-----------
cannot pre-compute NowBOp
Modified Paths:
--------------
branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
Modified: branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-18 17:51:05 UTC (rev 8620)
+++ branches/SESAME_2_7/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2014-08-25 14:40:28 UTC (rev 8621)
@@ -90,6 +90,7 @@
import com.bigdata.rdf.internal.constraints.INeedsMaterialization.Requirement;
import com.bigdata.rdf.internal.constraints.InBOp;
import com.bigdata.rdf.internal.constraints.IsBoundBOp;
+import com.bigdata.rdf.internal.constraints.NowBOp;
import com.bigdata.rdf.internal.constraints.ProjectedConstraint;
import com.bigdata.rdf.internal.constraints.SPARQLConstraint;
import com.bigdata.rdf.internal.constraints.TryBeforeMaterializationConstraint;
@@ -4624,7 +4625,7 @@
}
- if (op instanceof UUIDBOp) {
+ if (op instanceof UUIDBOp || op instanceof NowBOp) {
/*
* We cannot pre-generate these, they need to be unique
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2014-08-18 17:51:06
|
Revision: 8620
http://sourceforge.net/p/bigdata/code/8620
Author: mrpersonick
Date: 2014-08-18 17:51:05 +0000 (Mon, 18 Aug 2014)
Log Message:
-----------
Fixed DTE CI error related to Ticket #1001.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestDTE.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestDTE.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestDTE.java 2014-08-16 14:33:51 UTC (rev 8619)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestDTE.java 2014-08-18 17:51:05 UTC (rev 8620)
@@ -60,6 +60,10 @@
for(DTE e : DTE.values()) {
+ if (e == DTE.Extension) {
+ continue;
+ }
+
// verify can decode from [v].
{
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-16 14:33:59
|
Revision: 8619
http://sourceforge.net/p/bigdata/code/8619
Author: thompsonbry
Date: 2014-08-16 14:33:51 +0000 (Sat, 16 Aug 2014)
Log Message:
-----------
This has been traced to a query hint override to control the operator parallelism (maxParallel=2). While the HashIndexOp does c
Workaround: Bind the maxParallel query hint more closely to a specific join or other operator (sometimes difficult or impossible).
Workaround: Do not use the maxParallel query hint if the query has solution set hash joins (which appear if there is a nested complex optional join group, sub-select, INCLUDE, etc.)
Fix: Added ISingleThreadedOp interface and modified PipelineOp.getMaxParallel() to test for that interface and ignore annotation when the interface is present on an operator. This interface was added to the following operators:
- HashIndexOp
- HTreeNamedSubqueryOp
- JVMNamedSubqueryOp
- HTreeHashJoinOp
- JVMHashJoinOp
- HTreeDistinctBindingSetsOp (also fixed unit tests that looked for UnsupportedOperationException rather than IllegalArgumentException)
- PipelinedAggregationOp (also fixed unit tests that looked for UnsupportedOperationException rather than IllegalArgumentException)
Other changes to:
- PipelineOp (added interface test and assertMaxParallelOne() method).
Note: Also reduced the #of threads for StressTestConcurrentUnisolatedIndices since it was failing in CI with "unable to create new native thread".
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/HTreeNamedSubqueryOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/JVMNamedSubqueryOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashJoinOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/HTreeDistinctBindingSetsOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/PipelinedAggregationOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/join/TestJVMHashJoinOp.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestHTreeDistinctBindingSets.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/ISingleThreadedOp.java
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/ISingleThreadedOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/ISingleThreadedOp.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/ISingleThreadedOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -0,0 +1,40 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Aug 26, 2010
+ */
+package com.bigdata.bop;
+
+/**
+ * Marker interface for an operator whose instances do not support concurrent
+ * execution.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ *
+ * @see PipelineOp.Annotations#MAX_PARALLEL
+ */
+public interface ISingleThreadedOp {
+
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/PipelineOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -122,6 +122,8 @@
* {@link #MAX_MESSAGES_PER_TASK} and {@link #PIPELINE_QUEUE_CAPACITY}
* have less effect and performance tends to be best around a modest
* value (10) for those annotations.
+ *
+ * @see ISingleThreadedOp
*/
String MAX_PARALLEL = PipelineOp.class.getName() + ".maxParallel";
@@ -505,17 +507,49 @@
//
// }
- /**
- * The maximum parallelism with which tasks may be evaluated for this
- * operator (this is a per-shard limit in scale-out). A value of ONE (1)
- * indicates that at most ONE (1) instance of this task may be executing in
- * parallel for a given shard and may be used to indicate that the operator
- * evaluation task is not thread-safe.
- *
- * @see Annotations#MAX_PARALLEL
- */
+ /**
+ * If parallel evaluation is not allowed, then throws
+ * {@link IllegalArgumentException}.
+ */
+ final protected void assertMaxParallelOne() {
+
+ /*
+ * Note: Tests the annotation, not getMaxParallel(), since we want to
+ * make sure the annotation is valid and getMaxParallel() also tests for
+ * the ISingleThreadedOp interface.
+ */
+ if (getProperty(PipelineOp.Annotations.MAX_PARALLEL,
+ PipelineOp.Annotations.DEFAULT_MAX_PARALLEL) != 1) {
+
+ throw new IllegalArgumentException(
+ PipelineOp.Annotations.MAX_PARALLEL + "="
+ + getMaxParallel());
+
+ }
+
+ }
+
+ /**
+ * The maximum parallelism with which tasks may be evaluated for this
+ * operator (this is a per-shard limit in scale-out). A value of ONE (1)
+ * indicates that at most ONE (1) instance of this task may be executing in
+ * parallel for a given shard and may be used to indicate that the operator
+ * evaluation task is not thread-safe.
+ *
+ * @see Annotations#MAX_PARALLEL
+ * @see ISingleThreadedOp
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/1002"> </a>
+ */
final public int getMaxParallel() {
+ if(this instanceof ISingleThreadedOp) {
+
+ // Ignore the annotation value.
+ return 1;
+
+ }
+
return getProperty(PipelineOp.Annotations.MAX_PARALLEL,
PipelineOp.Annotations.DEFAULT_MAX_PARALLEL);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/HTreeNamedSubqueryOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/HTreeNamedSubqueryOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/HTreeNamedSubqueryOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -40,6 +40,7 @@
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IQueryAttributes;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
@@ -73,7 +74,8 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
-public class HTreeNamedSubqueryOp extends PipelineOp implements INamedSubqueryOp {
+public class HTreeNamedSubqueryOp extends PipelineOp implements
+ INamedSubqueryOp, ISingleThreadedOp {
static private final transient Logger log = Logger
.getLogger(HTreeNamedSubqueryOp.class);
@@ -123,11 +125,7 @@
+ getEvaluationContext());
}
- if (getMaxParallel() != 1) {
- throw new IllegalArgumentException(
- PipelineOp.Annotations.MAX_PARALLEL + "="
- + getMaxParallel());
- }
+ assertMaxParallelOne();
if (!isAtOnceEvaluation())
throw new IllegalArgumentException();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/JVMNamedSubqueryOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/JVMNamedSubqueryOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/controller/JVMNamedSubqueryOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -40,6 +40,7 @@
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IQueryAttributes;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
@@ -73,7 +74,8 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
-public class JVMNamedSubqueryOp extends PipelineOp implements INamedSubqueryOp {
+public class JVMNamedSubqueryOp extends PipelineOp implements INamedSubqueryOp,
+ ISingleThreadedOp {
static private final transient Logger log = Logger
.getLogger(JVMNamedSubqueryOp.class);
@@ -112,11 +114,7 @@
+ getEvaluationContext());
}
- if (getMaxParallel() != 1) {
- throw new IllegalArgumentException(
- PipelineOp.Annotations.MAX_PARALLEL + "="
- + getMaxParallel());
- }
+ assertMaxParallelOne();
if (!isAtOnceEvaluation())
throw new IllegalArgumentException();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -33,6 +33,7 @@
import com.bigdata.bop.BOpContext;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
import com.bigdata.bop.controller.INamedSolutionSetRef;
@@ -94,9 +95,9 @@
* @see HTreeHashJoinUtility
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
-public class HTreeHashJoinOp<E> extends HashJoinOp<E> {
+public class HTreeHashJoinOp<E> extends HashJoinOp<E> implements
+ ISingleThreadedOp {
/**
*
@@ -117,7 +118,7 @@
}
- public HTreeHashJoinOp(final BOp[] args, NV... annotations) {
+ public HTreeHashJoinOp(final BOp[] args, final NV... annotations) {
this(args, NV.asMap(annotations));
@@ -132,9 +133,7 @@
super(args, annotations);
- if (getMaxParallel() != 1)
- throw new UnsupportedOperationException(Annotations.MAX_PARALLEL
- + "=" + getMaxParallel());
+ assertMaxParallelOne();
// Note: This is no longer true. It is now shared via the IQueryAttributes.
// // shared state is used to share the hash table.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashIndexOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -38,6 +38,7 @@
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IQueryAttributes;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
@@ -76,7 +77,7 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
-abstract public class HashIndexOp extends PipelineOp {
+abstract public class HashIndexOp extends PipelineOp implements ISingleThreadedOp {
// static private final transient Logger log = Logger
// .getLogger(HashIndexOp.class);
@@ -144,15 +145,11 @@
// + getEvaluationContext());
// }
- if (getMaxParallel() != 1) {
- /*
- * Parallel evaluation is not allowed. This operator writes on an
- * object that is not thread-safe for mutation.
- */
- throw new IllegalArgumentException(
- PipelineOp.Annotations.MAX_PARALLEL + "="
- + getMaxParallel());
- }
+ /*
+ * This operator writes on an object that is not thread-safe for
+ * mutation.
+ */
+ assertMaxParallelOne();
if (!isLastPassRequested()) {
/*
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashJoinOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/HashJoinOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -65,7 +65,6 @@
* which join are output.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class HashJoinOp<E> extends PipelineOp implements
IShardwisePipelineOp<E> {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -34,6 +34,7 @@
import com.bigdata.bop.HashMapAnnotations;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.NV;
import com.bigdata.bop.controller.INamedSolutionSetRef;
import com.bigdata.relation.accesspath.IAccessPath;
@@ -56,9 +57,8 @@
* @see JVMHashJoinUtility
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
-public class JVMHashJoinOp<E> extends HashJoinOp<E> {
+public class JVMHashJoinOp<E> extends HashJoinOp<E> implements ISingleThreadedOp {
/**
*
@@ -94,9 +94,7 @@
super(args, annotations);
- if (getMaxParallel() != 1)
- throw new UnsupportedOperationException(Annotations.MAX_PARALLEL
- + "=" + getMaxParallel());
+ assertMaxParallelOne();
assertAtOnceJavaHeapOp();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/HTreeDistinctBindingSetsOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/HTreeDistinctBindingSetsOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/HTreeDistinctBindingSetsOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -9,6 +9,7 @@
import com.bigdata.bop.HTreeAnnotations;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IQueryAttributes;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
@@ -42,10 +43,9 @@
* on the native heap and eventually the machine will begin to swap.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: DistinctElementFilter.java 3466 2010-08-27 14:28:04Z
- * thompsonbry $
*/
-public class HTreeDistinctBindingSetsOp extends PipelineOp {
+public class HTreeDistinctBindingSetsOp extends PipelineOp implements
+ ISingleThreadedOp {
// private final static transient Logger log = Logger
// .getLogger(DistinctBindingSetsWithHTreeOp.class);
@@ -96,9 +96,7 @@
+ getEvaluationContext());
}
- if (getMaxParallel() != 1)
- throw new UnsupportedOperationException(Annotations.MAX_PARALLEL
- + "=" + getMaxParallel());
+ assertMaxParallelOne();
// // shared state is used to share the hash table.
// if (!isSharedState()) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/PipelinedAggregationOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/PipelinedAggregationOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/solutions/PipelinedAggregationOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -19,6 +19,7 @@
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IConstraint;
+import com.bigdata.bop.ISingleThreadedOp;
import com.bigdata.bop.IValueExpression;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.PipelineOp;
@@ -58,10 +59,9 @@
* the operator can still be invoked multiple times).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: DistinctElementFilter.java 3466 2010-08-27 14:28:04Z
- * thompsonbry $
*/
-public class PipelinedAggregationOp extends GroupByOp {
+public class PipelinedAggregationOp extends GroupByOp implements
+ ISingleThreadedOp {
private final static transient Logger log = Logger
.getLogger(PipelinedAggregationOp.class);
@@ -136,14 +136,11 @@
+ "=" + isLastPassRequested());
}
- if (getMaxParallel() != 1) {
- /*
- * Note: The operator MUST be single threaded in order to receive
- * the isLastInvocation notice.
- */
- throw new UnsupportedOperationException(Annotations.MAX_PARALLEL
- + "=" + getMaxParallel());
- }
+ /*
+ * Note: The operator MUST be single threaded in order to receive the
+ * isLastInvocation notice.
+ */
+ assertMaxParallelOne();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/join/TestJVMHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/join/TestJVMHashJoinOp.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/join/TestJVMHashJoinOp.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -201,8 +201,8 @@
new NV(PipelineOp.Annotations.MAX_PARALLEL, 2),//
namedSet,//
}));
- fail("Expecting: " + UnsupportedOperationException.class);
- } catch (UnsupportedOperationException ex) {
+ fail("Expecting: " + IllegalArgumentException.class);
+ } catch (IllegalArgumentException ex) {
if (log.isInfoEnabled())
log.info("Ignoring expected exception: " + ex);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestHTreeDistinctBindingSets.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestHTreeDistinctBindingSets.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/solutions/TestHTreeDistinctBindingSets.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -41,8 +41,6 @@
* Unit tests for {@link HTreeDistinctBindingSetsOp}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: TestDistinctBindingSets.java 4259 2011-02-28 16:24:53Z
- * thompsonbry $
*/
public class TestHTreeDistinctBindingSets extends
AbstractDistinctSolutionsTestCase {
@@ -161,8 +159,8 @@
// new NV(PipelineOp.Annotations.MAX_PARALLEL,
// 1),//
}));
- fail("Expecting: "+UnsupportedOperationException.class);
- } catch(UnsupportedOperationException ex) {
+ fail("Expecting: "+IllegalArgumentException.class);
+ } catch(IllegalArgumentException ex) {
if(log.isInfoEnabled())
log.info("Ignoring expected exception: "+ex);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-08-13 13:48:32 UTC (rev 8618)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-08-16 14:33:51 UTC (rev 8619)
@@ -142,9 +142,9 @@
3, // 3,// nresources // 20
1, // minLocks
2, // 5 // maxLocks // 3
- 1000, //5000, // ntrials // 1000
+ 500, // ntrials // Note: fails in CI @ 1000 (java.lang.OutOfMemoryError: unable to create new native thread)
3, // keyLen
- 1000, // 1000, // nops
+ 1000, // nops
0.02d,// failureRate
0.10d // commitRate
);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-13 13:48:38
|
Revision: 8618
http://sourceforge.net/p/bigdata/code/8618
Author: thompsonbry
Date: 2014-08-13 13:48:32 +0000 (Wed, 13 Aug 2014)
Log Message:
-----------
Strengthened the return type for UnisolatedReadWriteIndex (I thought that this was in the last commit).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-08-12 14:08:13 UTC (rev 8617)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-08-13 13:48:32 UTC (rev 8618)
@@ -362,7 +362,7 @@
* if the argument is <code>null</code>.
*/
// Note: Exposed to HTree, at least for now.
- public static ReadWriteLock getReadWriteLock(final ICommitter btree) {
+ public static ReentrantReadWriteLock getReadWriteLock(final ICommitter btree) {
if (btree == null)
throw new IllegalArgumentException();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:08:21
|
Revision: 8617
http://sourceforge.net/p/bigdata/code/8617
Author: thompsonbry
Date: 2014-08-12 14:08:13 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
Override annotations.
Changed to the same pattern that is used by the HTree and Stream classes to obtain the writeLock().
Old:
final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
New:
final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
lock.lock();
This avoids an unnecessary object creation.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-08-12 14:06:09 UTC (rev 8616)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-08-12 14:08:13 UTC (rev 8617)
@@ -155,7 +155,6 @@
* several published papers.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class BTree extends AbstractBTree implements //ICommitter,
ICheckpointProtocol {// ILocalBTreeView {
@@ -167,18 +166,21 @@
}
+ @Override
final public long getNodeCount() {
return nnodes;
}
+ @Override
final public long getLeafCount() {
return nleaves;
}
+ @Override
final public long getEntryCount() {
return nentries;
@@ -201,6 +203,7 @@
* counter will assign values within a namespace defined by the partition
* identifier.
*/
+ @Override
public ICounter getCounter() {
ICounter counter = new Counter(this);
@@ -628,12 +631,14 @@
// }
// final private boolean readOnly;
+ @Override
final public long getLastCommitTime() {
return lastCommitTime;
}
+ @Override
final public long getRevisionTimestamp() {
if (readOnly)
@@ -643,6 +648,7 @@
}
+ @Override
final public void setLastCommitTime(final long lastCommitTime) {
if (lastCommitTime == 0L)
@@ -685,6 +691,7 @@
/**
* Return the {@link IDirtyListener}.
*/
+ @Override
final public IDirtyListener getDirtyListener() {
return listener;
@@ -696,6 +703,7 @@
*
* @param listener The listener.
*/
+ @Override
final public void setDirtyListener(final IDirtyListener listener) {
assertNotReadOnly();
@@ -847,6 +855,7 @@
*
* @see #load(IRawStore, long, boolean)
*/
+ @Override
final public long writeCheckpoint() {
// write checkpoint and return address of that checkpoint record.
@@ -859,6 +868,7 @@
*
* @see #load(IRawStore, long, boolean)
*/
+ @Override
final public Checkpoint writeCheckpoint2() {
assertNotTransient();
@@ -889,8 +899,10 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/343
* @see https://sourceforge.net/apps/trac/bigdata/ticket/440
*/
- final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
- try {
+// final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
+ final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
+ lock.lock();
+ try {
if (/* autoCommit && */needsCheckpoint()) {
@@ -1084,6 +1096,7 @@
}
+ @Override
final public Checkpoint getCheckpoint() {
if (checkpoint == null)
@@ -1093,18 +1106,21 @@
}
+ @Override
final public long getRecordVersion() {
- return recordVersion;
+ return recordVersion;
}
+ @Override
final public long getMetadataAddr() {
- return metadata.getMetadataAddr();
+ return metadata.getMetadataAddr();
}
+ @Override
final public long getRootAddr() {
return (root == null ? getCheckpoint().getRootAddr() : root
@@ -1293,9 +1309,10 @@
* @return The address of a {@link Checkpoint} record from which the btree
* may be reloaded.
*/
+ @Override
public long handleCommit(final long commitTime) {
- return writeCheckpoint2().getCheckpointAddr();
+ return writeCheckpoint2().getCheckpointAddr();
}
@@ -1316,6 +1333,7 @@
* and dropping indices vs removing the entries in an individual
* {@link BTree}.
*/
+ @Override
final public void removeAll() {
assertNotReadOnly();
@@ -1892,6 +1910,7 @@
private NodeFactory() {
}
+ @Override
public Leaf allocLeaf(final AbstractBTree btree, final long addr,
final ILeafData data) {
@@ -1899,6 +1918,7 @@
}
+ @Override
public Node allocNode(final AbstractBTree btree, final long addr,
final INodeData data) {
@@ -1926,12 +1946,14 @@
}
+ @Override
public long get() {
return btree.counter.get();
}
+ @Override
public long incrementAndGet() {
final long counter = btree.counter.incrementAndGet();
@@ -2035,12 +2057,14 @@
}
+ @Override
public long get() {
return wrap( src.get() );
}
+ @Override
public long incrementAndGet() {
return wrap(src.incrementAndGet());
@@ -2098,6 +2122,7 @@
/**
* Returns ONE (1).
*/
+ @Override
final public int getSourceCount() {
return 1;
@@ -2107,24 +2132,28 @@
/**
* An array containing this {@link BTree}.
*/
+ @Override
final public AbstractBTree[] getSources() {
return new AbstractBTree[]{this};
}
+ @Override
final public BTree getMutableBTree() {
return this;
}
+ @Override
public LeafCursor newLeafCursor(final SeekEnum where) {
return new LeafCursor(where);
}
+ @Override
public LeafCursor newLeafCursor(final byte[] key) {
return new LeafCursor(key);
@@ -2400,18 +2429,21 @@
*/
private Leaf leaf;
+ @Override
public Leaf leaf() {
return leaf;
}
+ @Override
public BTree getBTree() {
return BTree.this;
}
+ @Override
public LeafCursor clone() {
return new LeafCursor(this);
@@ -2464,6 +2496,7 @@
}
+ @Override
public Leaf first() {
stack.clear();
@@ -2485,6 +2518,7 @@
}
+ @Override
public Leaf last() {
stack.clear();
@@ -2511,6 +2545,7 @@
* the leaf may not actually contain the key, in which case it is the
* leaf that contains the insertion point for the key.
*/
+ @Override
public Leaf seek(final byte[] key) {
stack.clear();
@@ -2533,6 +2568,7 @@
}
+ @Override
public Leaf seek(final ILeafCursor<Leaf> src) {
if (src == null)
@@ -2558,6 +2594,7 @@
}
+ @Override
public Leaf next() {
// make sure that the current leaf is valid.
@@ -2658,6 +2695,7 @@
* @return The prior leaf -or- <code>null</code> if there is no
* predecessor of this leaf.
*/
+ @Override
public Leaf prior() {
// make sure that the current leaf is valid.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:06:20
|
Revision: 8616
http://sourceforge.net/p/bigdata/code/8616
Author: thompsonbry
Date: 2014-08-12 14:06:09 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
Enabling StressTestUnisolatedReadWriteIndex in CI (see #355).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2014-08-12 14:05:34 UTC (rev 8615)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2014-08-12 14:06:09 UTC (rev 8616)
@@ -128,7 +128,7 @@
// test suite for hierarchical locking (namespace prefixes).
suite.addTestSuite(TestHierarchicalLockingTasks.class);
// test suite for GIST operations using group commit.
-// suite.addTestSuite(TestGISTTasks.class);
+ //suite.addTestSuite(TestGISTTasks.class);
// stress test of throughput when lock contention serializes unisolated writers.
suite.addTestSuite(StressTestLockContention.class);
// stress test of group commit.
@@ -138,11 +138,8 @@
/*
* Stress tests of writes on unisolated named indices using
* UnisolatedReadWriteIndex.
- *
- * FIXME This test appears to cause #343 (Stochastic assert in
- * AbstractBTree#writeNodeOrLeaf() in CI)
*/
-// suite.addTestSuite(StressTestUnisolatedReadWriteIndex.class);
+ suite.addTestSuite(StressTestUnisolatedReadWriteIndex.class);
/*
* Stress test of concurrent transactions.
*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:05:39
|
Revision: 8615
http://sourceforge.net/p/bigdata/code/8615
Author: thompsonbry
Date: 2014-08-12 14:05:34 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
javadoc.
Set timeout to Long.MAX_VALUE as fix for #355.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-08-12 14:05:05 UTC (rev 8614)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/journal/StressTestUnisolatedReadWriteIndex.java 2014-08-12 14:05:34 UTC (rev 8615)
@@ -127,13 +127,18 @@
* number of spurious extensions from the failureRate. However,
* there are clearly problems which emerge when the timeout is less
* than the time required to complete the scheduled tasks. A variety
- * of errors can be emerged when the scheduled tasks are all
+ * of errors can emerged when the scheduled tasks are all
* cancelled. It is difficult to say whether any of those problems
* could be observed by an application outside of a shutdownNow()
* scenario.
+ *
+ * TODO We could write this using a task queue feeding an executor
+ * pool so we could make it into a longer running test. If I simply
+ * increase the number of trials, it attempts to schedule them all
+ * concurrently and hits an out of memory error (too many native threads).
*/
doConcurrentClientTest(journal,//
- 30,// timeout
+ Long.MAX_VALUE,// timeout : MUST BE INFINITE OR WILL HIT FALSE ERRORS.
3, // 3,// nresources // 20
1, // minLocks
2, // 5 // maxLocks // 3
@@ -449,7 +454,7 @@
@Override
public Void call() throws Exception {
- final IIndex[] indices = new IIndex[resource.length];
+ final UnisolatedReadWriteIndex[] indices = new UnisolatedReadWriteIndex[resource.length];
final Thread t = Thread.currentThread();
@@ -496,7 +501,27 @@
ndx.remove(key);
- }
+ }
+ /**
+ * FIXME Add a probability of a read-only operation, e.g.,
+ * lookup() or rangeIterator(key,val). The latter can also
+ * do chunked resolution. This will provide test coverage
+ * for the case where the close() of the iterator interrupts
+ * the producer. This happens especially in the case where a
+ * range iterator is used and the iterator is closed after
+ * the first result since the producer could still be
+ * running. If it is in the middle of evicting a dirty node
+ * from the write retention queue then that can leave the
+ * nodes and the queue in an inconsistent state. (We might
+ * need to write that test at the SAIL level since it is the
+ * AbstractedChunkedResolverator pattern for which we have
+ * observed the issue. Perhaps through a modification of the
+ * csem stress test.)
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/855">
+ * AssertionError: Child does not have persistent
+ * identity </a>
+ */
} // for( i : nops )
@@ -562,7 +587,6 @@
* {@link TestOptions#FAILURE_RATE}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class SpuriousException extends RuntimeException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:05:17
|
Revision: 8614
http://sourceforge.net/p/bigdata/code/8614
Author: thompsonbry
Date: 2014-08-12 14:05:05 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
Annotations. Also exposed the ReentrantReadWriteIndex (strengthened the return type).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-08-12 14:04:21 UTC (rev 8613)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-08-12 14:05:05 UTC (rev 8614)
@@ -118,8 +118,6 @@
* computing the fix point of a rule set) is significantly lower.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: UnisolatedReadWriteIndex.java 4054 2011-01-05 13:51:25Z
- * thompsonbry $
*/
public class UnisolatedReadWriteIndex implements IIndex, ILinearList {
@@ -230,7 +228,7 @@
*
* @return The acquired lock.
*/
- private Lock lock(final IIndexProcedure proc) {
+ private Lock lock(final IIndexProcedure<?> proc) {
if (proc == null)
throw new IllegalArgumentException();
@@ -274,7 +272,7 @@
* Canonicalizing mapping for the locks used to control access to the
* unisolated index.
*/
- static final private WeakHashMap<ICommitter, ReadWriteLock> locks = new WeakHashMap<ICommitter,ReadWriteLock>();
+ static final private WeakHashMap<ICommitter, ReentrantReadWriteLock> locks = new WeakHashMap<ICommitter,ReentrantReadWriteLock>();
/**
* The default capacity for iterator reads against the underlying index. The
@@ -371,7 +369,7 @@
synchronized (locks) {
- ReadWriteLock readWriteLock = locks.get(btree);
+ ReentrantReadWriteLock readWriteLock = locks.get(btree);
if (readWriteLock == null) {
@@ -386,24 +384,28 @@
}
+ @Override
public String toString() {
return getClass().getSimpleName() + "{" + ndx.toString() + "}";
}
+ @Override
public IndexMetadata getIndexMetadata() {
return ndx.getIndexMetadata();
}
+ @Override
public IResourceMetadata[] getResourceMetadata() {
return getIndexMetadata().getPartitionMetadata().getResources();
}
+ @Override
public CounterSet getCounters() {
return ndx.getCounters();
@@ -416,12 +418,14 @@
*
* @throws UnsupportedOperationException
*/
+ @Override
public ICounter getCounter() {
throw new UnsupportedOperationException();
}
+ @Override
public boolean contains(final Object key) {
final Lock lock = readLock();
@@ -438,6 +442,7 @@
}
+ @Override
public Object insert(final Object key, final Object value) {
final Lock lock = writeLock();
@@ -454,6 +459,7 @@
}
+ @Override
public Object lookup(final Object key) {
final Lock lock = readLock();
@@ -470,6 +476,7 @@
}
+ @Override
public Object remove(final Object key) {
final Lock lock = writeLock();
@@ -486,6 +493,7 @@
}
+ @Override
public boolean contains(final byte[] key) {
final Lock lock = readLock();
@@ -502,6 +510,7 @@
}
+ @Override
public byte[] lookup(final byte[] key) {
final Lock lock = readLock();
@@ -518,6 +527,7 @@
}
+ @Override
public byte[] insert(final byte[] key, final byte[] value) {
final Lock lock = writeLock();
@@ -534,6 +544,7 @@
}
+ @Override
public byte[] remove(final byte[] key) {
final Lock lock = writeLock();
@@ -550,6 +561,7 @@
}
+ @Override
public long rangeCount() {
final Lock lock = readLock();
@@ -566,6 +578,7 @@
}
+ @Override
public long rangeCount(final byte[] fromKey, final byte[] toKey) {
final Lock lock = readLock();
@@ -582,6 +595,7 @@
}
+ @Override
public long rangeCountExact(final byte[] fromKey, final byte[] toKey) {
final Lock lock = readLock();
@@ -598,6 +612,7 @@
}
+ @Override
public long rangeCountExactWithDeleted(final byte[] fromKey, final byte[] toKey) {
final Lock lock = readLock();
@@ -614,12 +629,16 @@
}
+ @Override
+ @SuppressWarnings("rawtypes")
final public ITupleIterator rangeIterator() {
return rangeIterator(null, null);
}
+ @Override
+ @SuppressWarnings("rawtypes")
public ITupleIterator rangeIterator(final byte[] fromKey, final byte[] toKey) {
return rangeIterator(fromKey, toKey, 0/* capacity */,
@@ -636,6 +655,8 @@
* from the underlying index. Likewise, the mutation methods on the iterator
* will acquire the exclusive write lock.
*/
+ @Override
+ @SuppressWarnings("rawtypes")
public ITupleIterator rangeIterator(final byte[] fromKey, final byte[] toKey,
int capacity, int flags, final IFilter filter) {
@@ -681,7 +702,6 @@
* for the {@link Lock}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- *
*/
private class ChunkedIterator<E> extends ChunkedLocalRangeIterator<E> {
@@ -757,8 +777,9 @@
}
- }
+ } // ChunkedIterator
+ @Override
public Object submit(final byte[] key, final ISimpleIndexProcedure proc) {
final Lock lock = lock(proc);
@@ -780,6 +801,8 @@
}
+ @Override
+ @SuppressWarnings("rawtypes")
public void submit(final byte[] fromKey, final byte[] toKey,
final IKeyRangeIndexProcedure proc, final IResultHandler handler) {
@@ -802,7 +825,8 @@
}
- @SuppressWarnings("unchecked")
+ @Override
+ @SuppressWarnings({ "rawtypes", "unchecked" })
public void submit(final int fromIndex, final int toIndex, final byte[][] keys,
final byte[][] vals, final AbstractKeyArrayIndexProcedureConstructor ctor,
final IResultHandler aggregator) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:04:34
|
Revision: 8613
http://sourceforge.net/p/bigdata/code/8613
Author: thompsonbry
Date: 2014-08-12 14:04:21 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
removed version, added Override
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-08-12 14:04:04 UTC (rev 8612)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-08-12 14:04:21 UTC (rev 8613)
@@ -33,11 +33,11 @@
* persistence store.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class DefaultEvictionListener implements
IEvictionListener {
+ @Override
public void evicted(final IHardReferenceQueue<PO> cache, final PO ref) {
final AbstractNode<?> node = (AbstractNode<?>) ref;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-08-12 14:04:06
|
Revision: 8612
http://sourceforge.net/p/bigdata/code/8612
Author: thompsonbry
Date: 2014-08-12 14:04:04 +0000 (Tue, 12 Aug 2014)
Log Message:
-----------
javadoc related to #355 (closed, invalid).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-08-08 17:22:00 UTC (rev 8611)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-08-12 14:04:04 UTC (rev 8612)
@@ -3623,7 +3623,24 @@
assert node.dirty;
assert !node.deleted;
assert !node.isPersistent();
- assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
+ /**
+ * Occasional CI errors on this assert for have been observed for
+ * StressTestUnisolatedReadWriteIndex. This has been traced to a test
+ * error. The test was interrupting the tasks, but the tasks were not
+ * being cancelled simultaneously. This meant that one task could be
+ * interrupted during an eviction from the write retention queue and
+ * that another task could obtain the UnisolatedReadWriteIndex lock and
+ * then hit the error since the BTree, the write retention queue, and
+ * the nodes that were being evicted would an inconsistent state state.
+ * The test does not fail if it is run to completion (no timeout).
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/343" >Stochastic assert
+ * in AbstractBTree#writeNodeOrLeaf() in CI </a>
+ *
+ * TestMROWTransactions might also demonstrate an issue
+ * occasionally. If so, then check for the same root cause.
+ */
+ assert !node.isReadOnly();
assertNotReadOnly();
/*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-08-08 17:22:07
|
Revision: 8611
http://sourceforge.net/p/bigdata/code/8611
Author: tobycraig
Date: 2014-08-08 17:22:00 +0000 (Fri, 08 Aug 2014)
Log Message:
-----------
JSHint fixes
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-08 17:20:45 UTC (rev 8610)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-08 17:22:00 UTC (rev 8611)
@@ -1,6 +1,7 @@
$(function() {
'use strict';
+/* jshint validthis: true */
/* Global variables */
@@ -695,7 +696,7 @@
var url = RW_URL_PREFIX + 'namespace/' + NAMESPACE + '/sparql';
var settings = {
type: 'POST',
- data: FILE_CONTENTS == null ? EDITORS.update.getValue() : FILE_CONTENTS,
+ data: FILE_CONTENTS === null ? EDITORS.update.getValue() : FILE_CONTENTS,
success: updateResponseXML,
error: updateResponseError
};
@@ -782,7 +783,7 @@
if(jqXHR.status === 0) {
message += 'Could not contact server';
} else {
- if(response.find('pre').length == 0) {
+ if(response.find('pre').length === 0) {
message += response.text();
} else {
message += response.find('pre').text();
@@ -1189,7 +1190,7 @@
message += 'Could not contact server';
} else {
var response = $('<div>').append(jqXHR.responseText);
- if(response.find('pre').length == 0) {
+ if(response.find('pre').length === 0) {
message += response.text();
} else {
message += response.find('pre').text();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-08-08 17:20:54
|
Revision: 8610
http://sourceforge.net/p/bigdata/code/8610
Author: tobycraig
Date: 2014-08-08 17:20:45 +0000 (Fri, 08 Aug 2014)
Log Message:
-----------
Made running queries in history clickable to show query status
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-08 00:33:42 UTC (rev 8609)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-08-08 17:20:45 UTC (rev 8610)
@@ -895,8 +895,8 @@
if($(row).find('.query')[0].innerText == query) {
// clear the old results and set the time to now
$(row).find('.query-time').text(new Date().toISOString());
- $(row).find('.query-results').text('...');
- $(row).find('.query-execution-time').text('...');
+ $(row).find('.query-results').text('');
+ $(row).find('.query-execution-time').html('<a href="#">Running...</a>');
// move it to the top
$(row).prependTo('#query-history tbody');
queryExists = true;
@@ -905,7 +905,7 @@
});
if(!queryExists) {
- addQueryHistoryRow(new Date().toISOString(), query, '...', '...', true);
+ addQueryHistoryRow(new Date().toISOString(), query, '', '<a href="#">Running...</a>', true);
}
storeQueryHistory();
@@ -1572,6 +1572,12 @@
$('p:contains(Show queries, query details)').find('a').eq(1).click(function(e) { e.preventDefault(); showQueries(true); });
}
+function showRunningQueries(e) {
+ e.preventDefault();
+ showTab('status');
+ showQueries();
+}
+
function showQueries(details) {
var url = RO_URL_PREFIX + 'status?showQueries';
if(details) {
@@ -1858,6 +1864,7 @@
$('#query-explain').change(handleExplain);
$('#query-details').change(handleDetails);
$('#query-history').on('click', '.query a', loadHistory);
+ $('#query-history').on('click', '.query-execution-time a', showRunningQueries);
$('#query-history').on('click', '.query-delete a', deleteHistoryRow);
$('#query-history-clear').click(clearHistory);
$('#query-response-clear').click(clearQueryResponse);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|