Menu
▾
▴
[ogsa-dai svn] SF.net SVN: ogsa-dai:[2075] sandbox/dqp/server/src
|
From: <amy...@us...> - 2012-07-16 11:05:45
|
Revision: 2075
http://ogsa-dai.svn.sourceforge.net/ogsa-dai/?rev=2075&view=rev
Author: amykrause
Date: 2012-07-16 11:05:38 +0000 (Mon, 16 Jul 2012)
Log Message:
-----------
Added Derby join builder.
Fixed bugs:
- convert physical schema into statistics schema if necessary
- join implementation optimiser raises proper exception if cardinality annotations are missing
Modified Paths:
--------------
sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityEstimator.java
sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityUtils.java
sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/optimiser/join/EquiJoinBatch.java
sandbox/dqp/server/src/test/unit/resources/DQPCompilerConfiguration.xml
Added Paths:
-----------
sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/execute/workflow/DerbyJoinBuilder.java
Added: sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/execute/workflow/DerbyJoinBuilder.java
===================================================================
--- sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/execute/workflow/DerbyJoinBuilder.java (rev 0)
+++ sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/execute/workflow/DerbyJoinBuilder.java 2012-07-16 11:05:38 UTC (rev 2075)
@@ -0,0 +1,83 @@
+// Copyright (c) The University of Edinburgh, 2012.
+//
+// LICENCE-START
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// LICENCE-END
+
+package uk.org.ogsadai.dqp.execute.workflow;
+
+import java.util.List;
+
+import uk.org.ogsadai.client.toolkit.SingleActivityOutput;
+import uk.org.ogsadai.client.toolkit.activities.generic.GenericActivity;
+import uk.org.ogsadai.dqp.execute.ActivityConstructionException;
+import uk.org.ogsadai.dqp.lqp.Operator;
+import uk.org.ogsadai.dqp.lqp.operators.InnerThetaJoinOperator;
+import uk.org.ogsadai.expression.Expression;
+import uk.org.ogsadai.expression.ExpressionUtils;
+
+/**
+ * Builds activities for operator INNER THETA JOIN. This builder creates a
+ * DerbyJoin activity and handles the renaming of the columns.
+ *
+ * @author The OGSA-DAI Project Team.
+ */
+public class DerbyJoinBuilder implements ActivityPipelineBuilder
+{
+ /** Copyright notice. */
+ private static final String COPYRIGHT_NOTICE =
+ "Copyright (c) The University of Edinburgh, 2012";
+
+ private String mResourceID;
+
+ /**
+ * Sets the resource ID of the data resource for storing temporary tables.
+ *
+ * @param resourceID
+ * data resource ID
+ */
+ public void setResourceID(String resourceID)
+ {
+ mResourceID = resourceID;
+ }
+
+ @Override
+ public SingleActivityOutput build(
+ Operator op,
+ List<SingleActivityOutput> outputs,
+ PipelineWorkflowBuilder builder)
+ throws ActivityConstructionException
+ {
+ InnerThetaJoinOperator operator = (InnerThetaJoinOperator)op;
+ SingleActivityOutput outputLeft = outputs.get(0);
+ SingleActivityOutput outputRight = outputs.get(1);
+
+ Expression expression = operator.getPredicate().getExpression();
+ String sql = ExpressionUtils.generateSQL(expression);
+
+ GenericActivity derbyJoin =
+ new GenericActivity("uk.org.ogsadai.DerbyJoin");
+ derbyJoin.setResourceID(mResourceID);
+ derbyJoin.createInput("data1");
+ derbyJoin.createInput("data2");
+ derbyJoin.createInput("condition");
+ derbyJoin.createOutput("result");
+ derbyJoin.addInput("condition", sql);
+ derbyJoin.connectInput("data1", outputLeft);
+ derbyJoin.connectInput("data2", outputRight);
+ builder.add(derbyJoin);
+
+ return derbyJoin.getOutput("result");
+ }
+
+}
\ No newline at end of file
Modified: sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityEstimator.java
===================================================================
--- sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityEstimator.java 2012-07-16 09:57:57 UTC (rev 2074)
+++ sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityEstimator.java 2012-07-16 11:05:38 UTC (rev 2075)
@@ -19,6 +19,7 @@
import java.util.List;
import java.util.Map.Entry;
+import uk.org.ogsadai.common.msgs.DAILogger;
import uk.org.ogsadai.dqp.common.DataDictionary;
import uk.org.ogsadai.dqp.common.PhysicalSchema;
import uk.org.ogsadai.dqp.lqp.Annotation;
@@ -66,6 +67,9 @@
/** Copyright notice */
private static final String COPYRIGHT_NOTICE =
"Copyright (c) The University of Edinburgh, 2012.";
+
+ private static final DAILogger LOG =
+ DAILogger.getLogger(CardinalityEstimator.class);
private DataDictionary mDataDictionary;
private ApplyOperator mWrapperApplyOperator;
@@ -264,8 +268,11 @@
mDataDictionary.getTableSchema(
operator.getTableName()).getPhysicalSchema();
- if (!(physicalSchema instanceof StatisticsPhysicalSchema))
+ if (physicalSchema == null)
{
+ LOG.debug("No physical schema found for table " +
+ operator.getTableName());
+ // make up a schema?
CardinalityStatistics cardStats =
CardinalityUtils.makeStatisticsFromHeading(
operator.getHeading().getAttributes(),
@@ -275,6 +282,21 @@
// "No physical schema data for table : " +
// operator.getTableName());
}
+ else if (!(physicalSchema instanceof StatisticsPhysicalSchema))
+ {
+ LOG.debug("Simple physical schema found for table " +
+ operator.getTableName());
+ LOG.debug("Type = " + physicalSchema.getClass().getName());
+ long cardinality = physicalSchema.getCardinality();
+ LOG.debug("Cardinality = " + cardinality);
+ // Convert simple physical schema into statistics schema.
+ CardinalityStatistics cardStats =
+ CardinalityUtils.makeStatisticsFromHeading(
+ operator.getHeading().getAttributes(),
+ cardinality,
+ cardinality/10.0);
+ addStatistics(operator, cardStats);
+ }
else
{
StatisticsPhysicalSchema statsPhysicalSchema =
Modified: sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityUtils.java
===================================================================
--- sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityUtils.java 2012-07-16 09:57:57 UTC (rev 2074)
+++ sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/cardinality/CardinalityUtils.java 2012-07-16 11:05:38 UTC (rev 2075)
@@ -537,6 +537,10 @@
new ScalarAttributeStatistics(numRows, numValues);
for (Attribute attr: heading)
{
+ if (attr.isKey())
+ {
+ stats = new ScalarAttributeStatistics(numRows, numRows);
+ }
result.addAttributeStatistics(attr, stats);
}
return result;
Modified: sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/optimiser/join/EquiJoinBatch.java
===================================================================
--- sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/optimiser/join/EquiJoinBatch.java 2012-07-16 09:57:57 UTC (rev 2074)
+++ sandbox/dqp/server/src/main/java/uk/org/ogsadai/dqp/lqp/optimiser/join/EquiJoinBatch.java 2012-07-16 11:05:38 UTC (rev 2075)
@@ -28,10 +28,11 @@
if (suitableTableScans.size() == 0) return null;
- Double[] card = {
- Annotation.getCardinalityAnnotation(joinOperator.getChild(0)),
- Annotation.getCardinalityAnnotation(joinOperator.getChild(1)) };
-
+ Double[] card = {
+ getChildCardinality(0, joinOperator),
+ getChildCardinality(1, joinOperator)
+ };
+
// To create as few batch queries as possible we wish to stream on
// the side with the lowest cardinality and hence we repeatedly query
// the side with the largest cardinality.
@@ -67,6 +68,19 @@
}
+ private Double getChildCardinality(int index, Operator operator)
+ {
+ Double childCard =
+ Annotation.getCardinalityAnnotation(operator.getChild(index));
+ if (childCard == null)
+ {
+ throw new RuntimeException(
+ "Operator " + operator.getChild(index) +
+ " has no CardinalityStatistics annotation");
+ }
+ return childCard;
+ }
+
/**
* Finds a list of table scan operators suitable for filtering for a given
* join operator.
Modified: sandbox/dqp/server/src/test/unit/resources/DQPCompilerConfiguration.xml
===================================================================
--- sandbox/dqp/server/src/test/unit/resources/DQPCompilerConfiguration.xml 2012-07-16 09:57:57 UTC (rev 2074)
+++ sandbox/dqp/server/src/test/unit/resources/DQPCompilerConfiguration.xml 2012-07-16 11:05:38 UTC (rev 2075)
@@ -14,16 +14,6 @@
<bean id="partitioner"
class="uk.org.ogsadai.dqp.lqp.optimiser.partitioner.PartitioningOptimiser"/>
- <bean id="equiJoinBatch"
- class="uk.org.ogsadai.dqp.lqp.optimiser.join.EquiJoinBatch">
- <property name="batchSize" value="1000" />
- </bean>
- <bean id="inMemoryJoin"
- class="uk.org.ogsadai.dqp.lqp.optimiser.join.InMemoryJoin"/>
- <bean id="orderedSweepJoin"
- class="uk.org.ogsadai.dqp.lqp.optimiser.join.OrderedSweepJoin"/>
-
-
<!-- **************************
Compiler Configuration
************************** -->
@@ -31,7 +21,6 @@
<bean id="compilerConfiguration"
class="uk.org.ogsadai.resource.dataresource.dqp.SimpleCompilerConfiguration">
-
<!--
This section specifies the chain of optimisers that will rewrite the
query plan. Optimisers will be chained in the specification order.
@@ -56,18 +45,21 @@
<bean class="uk.org.ogsadai.dqp.lqp.optimiser.implosion.TableScanImplosionOptimiser">
<property name="cardinalityEstimator" ref="cardinalityEstimator"/>
</bean>
- <!--
+ <!--
<bean class="uk.org.ogsadai.dqp.lqp.optimiser.implosion.ExtendedTableScanImplosionOptimiser">
<property name="partitioner" ref="partitioner"/>
<property name="cardinalityEstimator" ref="cardinalityEstimator"/>
</bean>
- -->
+ -->
<bean class="uk.org.ogsadai.dqp.lqp.optimiser.join.JoinImplementationOptimiser">
<property name="joinImplementations">
<list>
- <ref bean="equiJoinBatch"/>
- <ref bean="inMemoryJoin"/>
- <ref bean="orderedSweepJoin"/>
+ <bean class="uk.org.ogsadai.dqp.lqp.optimiser.join.EquiJoinBatch">
+ <property name="batchSize" value="1000" />
+ </bean>
+ <bean class="uk.org.ogsadai.dqp.lqp.optimiser.join.InMemoryJoin"/>
+ <bean class="uk.org.ogsadai.dqp.lqp.optimiser.join.OrderedSweepJoin"/>
+ <bean class="uk.org.ogsadai.dqp.lqp.optimiser.join.DerbyJoin"/>
</list>
</property>
</bean>
@@ -108,6 +100,11 @@
<entry key="PRIMARY_EXPRESSION"> <bean class="uk.org.ogsadai.dqp.execute.workflow.ThetaJoinBuilder"/> </entry>
<entry key="EquiJoinBatch"> <bean class="uk.org.ogsadai.dqp.execute.workflow.BatchJoinBuilder"/> </entry>
<entry key="PLANE_SWEEP"> <bean class="uk.org.ogsadai.dqp.execute.workflow.OrderedSweepJoinBuilder"/> </entry>
+ <entry key="DERBY_JOIN">
+ <bean class="uk.org.ogsadai.dqp.execute.workflow.DerbyJoinBuilder">
+ <property name="resourceID" value="DerbyResource"/>
+ </bean>
+ </entry>
</map>
</entry>
<entry key="LEFT_OUTER_JOIN">
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|